diff options
215 files changed, 5645 insertions, 2055 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c358e93e..36eab5027 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,15 @@ name: ci/gh-actions/cli on: [push, pull_request] +# The below variables reduce repetitions across similar targets +env: + REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost + APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache' + APT_SET_CONF: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + jobs: build-macos: runs-on: macOS-latest @@ -66,16 +75,13 @@ jobs: key: ccache-ubuntu-build-${{ github.sha }} restore-keys: ccache-ubuntu-build- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: build run: | ccache --max-size=150M @@ -96,16 +102,13 @@ jobs: key: ccache-ubuntu-libwallet-${{ github.sha }} restore-keys: ccache-ubuntu-libwallet- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: build run: | ccache --max-size=150M @@ -115,24 +118,56 @@ jobs: test-ubuntu: needs: build-ubuntu runs-on: ubuntu-latest + env: + CCACHE_COMPRESS: 1 + CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 with: submodules: recursive + - name: ccache + uses: actions/cache@v2 + with: + path: ~/.ccache + key: test-ubuntu-ccache-${{ github.sha }} + restore-keys: test-ubuntu-ccache- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler - - name: install requests - run: pip install requests + run: ${{env.APT_INSTALL_LINUX}} + - name: install Python dependencies + run: pip install requests psutil monotonic - name: tests env: CTEST_OUTPUT_ON_FAILURE: ON - run: make release-test -j3 + run: | + ccache --max-size=150M + DIR_BUILD="build/ci/release" + DIR_SRC="`pwd`" + mkdir -p "${DIR_BUILD}" && cd "${DIR_BUILD}" + cmake -S "${DIR_SRC}" -D ARCH="default" -D BUILD_SHARED_LIBS=ON -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release && make -j3 && make test + +# ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere. +# BUILD_SHARED_LIBS=ON speeds up the linkage part a bit, reduces size, and is the only place where the dynamic linkage is tested. + + source-archive: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: archive + run: | + pip install git-archive-all + export VERSION="monero-$(git describe)" + export OUTPUT="$VERSION.tar" + echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV + /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT" + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.OUTPUT }} + path: /home/runner/work/monero/monero/${{ env.OUTPUT }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 49ac18c66..f2cdc1818 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ if (IOS) INCLUDE(CmakeLists_IOS.txt) endif() -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.5) message(STATUS "CMake version ${CMAKE_VERSION}") project(monero) @@ -106,16 +106,18 @@ function (die msg) endfunction () function (add_c_flag_if_supported flag var) - string(REPLACE "-" "_" supported ${flag}_c) - check_c_compiler_flag(${flag} ${supported}) + set(TMP "-Werror ${flag}") + string(REGEX REPLACE "[- ]" "_" supported ${TMP}_c) + check_c_compiler_flag(${TMP} ${supported}) if(${${supported}}) set(${var} "${${var}} ${flag}" PARENT_SCOPE) endif() endfunction() function (add_cxx_flag_if_supported flag var) - string(REPLACE "-" "_" supported ${flag}_cxx) - check_cxx_compiler_flag(${flag} ${supported}) + set(TMP "-Werror ${flag}") + string(REGEX REPLACE "[- ]" "_" supported ${TMP}_cxx) + check_cxx_compiler_flag(${TMP} ${supported}) if(${${supported}}) set(${var} "${${var}} ${flag}" PARENT_SCOPE) endif() @@ -154,6 +156,15 @@ function (monero_set_target_no_relink target) endif() endfunction() +option(STRIP_TARGETS "Strip symbols from targets?" OFF) +function (monero_set_target_strip target) + if (STRIP_TARGETS) + set_target_properties("${target}" PROPERTIES LINK_FLAGS_RELEASE -s) + set_target_properties("${target}" PROPERTIES LINK_FLAGS_DEBUG -s) + # Stripping from Debug might make sense if you're low on disk space, but want to test if debug version builds properly. + endif() +endfunction() + function (monero_add_minimal_executable name) source_group("${name}" FILES @@ -161,9 +172,82 @@ function (monero_add_minimal_executable name) add_executable("${name}" ${ARGN}) - monero_set_target_no_relink( ${name} ) + monero_set_target_no_relink("${name}") + monero_set_target_strip ("${name}") endfunction() +# Finds all headers in a directory and its subdirs, to be able to search for them and autosave in IDEs. +# +# Parameters: +# - headers_found: Output variable, which will hold the found headers +# - module_root_dir: The search path for the headers. Typically it will be the module's root dir, so "${CMAKE_CURRENT_SOURCE_DIR}" or a derivative of it. +macro (monero_find_all_headers headers_found module_root_dir) + file(GLOB ${headers_found} + "${module_root_dir}/*.h*" # h* will include hpps as well. + "${module_root_dir}/**/*.h*" # Any number of subdirs will be included. + "${module_root_dir}/*.inl" # .inl is typically template code and is being treated as headers (it's being included). + "${module_root_dir}/**/*.inl" +) +endmacro() + +# Function to forbid undefined symbols and also verify +# 1) Test project with all types of libraries and without undefined symbols can compile successfully +# 2) Test project with all types of libraries and undefined symbols can not compile successfully +function(forbid_undefined_symbols) + unset(TMP) + # https://www.unix.com/man-page/linux/1/ld, --no-undefined, Report unresolved symbol references from regular object files. + add_linker_flag_if_supported(-Wl,--no-undefined TMP) + # https://www.unix.com/man-page/osx/1/ld/, -undefined, Specifies how undefined symbols are to be treated. + add_linker_flag_if_supported(-Wl,-undefined,error TMP) + string(APPEND CMAKE_SHARED_LINKER_FLAGS ${TMP}) + string(APPEND CMAKE_MODULE_LINKER_FLAGS ${TMP}) + set(CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} PARENT_SCOPE) + set(CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} PARENT_SCOPE) + set(TEST_PROJECT "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CMakeTmp/test_project") + foreach(EXPECT IN ITEMS TRUE FALSE) + file(REMOVE_RECURSE "${TEST_PROJECT}") + file(MAKE_DIRECTORY "${TEST_PROJECT}") + file(WRITE "${TEST_PROJECT}/CMakeLists.txt" + [=[ +cmake_minimum_required(VERSION 3.1) +project(test) +option(EXPECT_SUCCESS "" ON) +file(WRITE "${CMAKE_SOURCE_DIR}/incorrect_source.cpp" "void undefined_symbol(); void symbol() { undefined_symbol(); }") +if (EXPECT_SUCCESS) + file(APPEND "${CMAKE_SOURCE_DIR}/incorrect_source.cpp" " void undefined_symbol() {}; ") +endif() +add_library(l0 SHARED incorrect_source.cpp) +add_library(l1 MODULE incorrect_source.cpp) +add_library(l2 STATIC incorrect_source.cpp) +add_library(l3 OBJECT incorrect_source.cpp) +]=] + ) + try_compile(SUCCESS "${TEST_PROJECT}/build" "${TEST_PROJECT}" test + CMAKE_FLAGS + "-DCMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS}" + "-DCMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS}" + "-DEXPECT_SUCCESS=${EXPECT}" + ) + if (NOT ${SUCCESS} STREQUAL ${EXPECT}) + message(FATAL_ERROR "Undefined symbols test failure: expect(${EXPECT}), success(${SUCCESS})") + endif() + file(REMOVE_RECURSE "${TEST_PROJECT}") + endforeach() +endfunction() +forbid_undefined_symbols() + +if (MINGW) + function(export_all_symbols) + unset(TMP) + add_linker_flag_if_supported(-Wl,--export-all-symbols TMP) + string(APPEND CMAKE_SHARED_LINKER_FLAGS ${TMP}) + string(APPEND CMAKE_MODULE_LINKER_FLAGS ${TMP}) + set(CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} PARENT_SCOPE) + set(CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} PARENT_SCOPE) + endfunction() + export_all_symbols() +endif() + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}") @@ -484,7 +568,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "(SunOS|Solaris)") endif () if (APPLE AND NOT IOS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64 -fvisibility=default -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default -std=c++11") if (NOT OPENSSL_ROOT_DIR) EXECUTE_PROCESS(COMMAND brew --prefix openssl OUTPUT_VARIABLE OPENSSL_ROOT_DIR @@ -502,6 +586,10 @@ if(STATIC AND NOT IOS) endif() endif() +if (WIN32) + list(APPEND OPENSSL_LIBRARIES ws2_32 crypt32) +endif() + find_package(HIDAPI) add_definition_if_library_exists(c memset_s "string.h" HAVE_MEMSET_S) @@ -513,6 +601,44 @@ add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP) set(MONERO_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_include") include_directories(${MONERO_GENERATED_HEADERS_DIR}) +option(COVERAGE "Enable profiling for test coverage report" OFF) +if(COVERAGE) + message(STATUS "Building with profiling for test coverage report") +endif() +macro (monero_enable_coverage) + if(COVERAGE) + foreach(COV_FLAG -fprofile-arcs -ftest-coverage --coverage) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COV_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COV_FLAG}") + endforeach() + endif() +endmacro() + +function (monero_add_library name) + monero_add_library_with_deps(NAME "${name}" SOURCES ${ARGN}) +endfunction() + +function (monero_add_library_with_deps) + cmake_parse_arguments(MONERO_ADD_LIBRARY "" "NAME" "DEPENDS;SOURCES" ${ARGN}) + source_group("${MONERO_ADD_LIBRARY_NAME}" FILES ${MONERO_ADD_LIBRARY_SOURCES}) + + # Define a ("virtual") object library and an actual library that links those + # objects together. The virtual libraries can be arbitrarily combined to link + # any subset of objects into one library archive. This is used for releasing + # libwallet, which combines multiple components. + set(objlib obj_${MONERO_ADD_LIBRARY_NAME}) + add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES}) + add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>) + monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}") + monero_set_target_strip ("${MONERO_ADD_LIBRARY_NAME}") + if (MONERO_ADD_LIBRARY_DEPENDS) + add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS}) + endif() + set_property(TARGET "${MONERO_ADD_LIBRARY_NAME}" PROPERTY FOLDER "libs") + target_compile_definitions(${objlib} + PRIVATE $<TARGET_PROPERTY:${MONERO_ADD_LIBRARY_NAME},INTERFACE_COMPILE_DEFINITIONS>) +endfunction () + # Generate header for embedded translations # Generate header for embedded translations, use target toolchain if depends, otherwise use the # lrelease and lupdate binaries from the host @@ -554,6 +680,17 @@ endif() # Trezor support check include(CheckTrezor) +# As of OpenBSD 6.8, -march=<anything> breaks the build +function(set_default_arch) + if (OPENBSD) + set(ARCH default) + else() + set(ARCH native) + endif() + + set(ARCH ${ARCH} CACHE STRING "CPU to build for: -march value or 'default' to not pass -march at all") +endfunction() + if(MSVC) add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__") # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline") @@ -567,7 +704,7 @@ if(MSVC) else() include(TestCXXAcceptsFlag) if (NOT ARCH) - set(ARCH native CACHE STRING "CPU to build for: -march value or 'default' to not pass -march at all") + set_default_arch() endif() message(STATUS "Building on ${CMAKE_SYSTEM_PROCESSOR} for ${ARCH}") if(ARCH STREQUAL "default") @@ -630,7 +767,7 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAG}") set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") - if(CMAKE_C_COMPILER_ID STREQUAL "Clang") + if(CMAKE_C_COMPILER_ID MATCHES "Clang") if(ARM) set(WARNINGS "${WARNINGS} -Wno-error=inline-asm") endif() @@ -665,13 +802,7 @@ else() set(STATIC_ASSERT_CPP_FLAG "-Dstatic_assert=_Static_assert") endif() - option(COVERAGE "Enable profiling for test coverage report" 0) - - if(COVERAGE) - message(STATUS "Building with profiling for test coverage report") - set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage --coverage") - endif() - + monero_enable_coverage() # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that # is fixed in the code (Issue #847), force compiler to be conservative. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") @@ -718,7 +849,12 @@ else() # PIE executables randomly crash at startup with ASAN # Windows binaries die on startup with PIE when compiled with GCC <9.x # Windows dynamically-linked binaries die on startup with PIE regardless of GCC version - add_linker_flag_if_supported(-pie LD_SECURITY_FLAGS) + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + # Clang does not support -pie flag + add_linker_flag_if_supported("-Wl,-pie" LD_SECURITY_FLAGS) + else() + add_linker_flag_if_supported("-pie" LD_SECURITY_FLAGS) + endif() endif() add_linker_flag_if_supported(-Wl,-z,relro LD_SECURITY_FLAGS) add_linker_flag_if_supported(-Wl,-z,now LD_SECURITY_FLAGS) @@ -744,12 +880,19 @@ else() add_linker_flag_if_supported(-Wl,--high-entropy-va LD_SECURITY_FLAGS) endif() + # Warnings, that when ignored are so severe, that they can segfault or even UB any application. + # Treat them as errors. + add_c_flag_if_supported( -Werror=switch C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-Werror=switch CXX_SECURITY_FLAGS) + add_c_flag_if_supported( -Werror=return-type C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-Werror=return-type CXX_SECURITY_FLAGS) + message(STATUS "Using C security hardening flags: ${C_SECURITY_FLAGS}") message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}") # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that @@ -867,7 +1010,7 @@ else() endif() set(USE_LTO ${USE_LTO_DEFAULT} CACHE BOOL "Use Link-Time Optimization (Release mode only)") - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") # There is a clang bug that does not allow to compile code that uses AES-NI intrinsics if -flto is enabled, so explicitly disable set(USE_LTO false) endif() @@ -1014,7 +1157,7 @@ if(ANDROID) set(ATOMIC libatomic.a) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=user-defined-warnings") endif() -if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AND NOT FREEBSD) +if(CMAKE_C_COMPILER_ID MATCHES "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AND NOT FREEBSD) find_library(ATOMIC atomic) if (ATOMIC_FOUND) list(APPEND EXTRA_LIBRARIES ${ATOMIC}) diff --git a/Dockerfile b/Dockerfile index 51fd51e1c..f21f74ac3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ ARG BOOST_VERSION=1_70_0 ARG BOOST_VERSION_DOT=1.70.0 ARG BOOST_HASH=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ + && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOT}/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && cd boost_${BOOST_VERSION} \ @@ -32,7 +32,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - Forum: [forum.getmonero.org](https://forum.getmonero.org) - Mail: [dev@getmonero.org](mailto:dev@getmonero.org) - GitHub: [https://github.com/monero-project/monero](https://github.com/monero-project/monero) -- IRC: [#monero-dev on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-dev&prompt=1&uio=d4) +- IRC: [#monero-dev on Libera](https://web.libera.chat/#monero-dev) - It is HIGHLY recommended that you join the #monero-dev IRC channel if you are developing software that uses Monero. Due to the nature of this open source software project, joining this channel and idling is the best way to stay updated on best practices and new developments in the Monero ecosystem. All you need to do is join the IRC channel and idle to stay updated with the latest in Monero development. If you do not, you risk wasting resources on developing integrations that are not compatible with the Monero network. The Monero core team and community continuously make efforts to communicate updates, developments, and documentation via other platforms – but for the best information, you need to talk to other Monero developers, and they are on IRC. #monero-dev is about Monero development, not getting help about using Monero, or help about development of other software, including yours, unless it also pertains to Monero code itself. For these cases, checkout #monero. ## Vulnerability response @@ -44,7 +44,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. -Our researchers are available on IRC in [#monero-research-lab on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-research-lab&prompt=1&uio=d4) or by email: +Our researchers are available on IRC in [#monero-research-lab on Libera](https://web.libera.chat/#monero-research-lab) or by email: - Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) - Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) @@ -92,7 +92,7 @@ As with many development projects, the repository on Github is considered to be Monero is a 100% community-sponsored endeavor. If you want to join our efforts, the easiest thing you can do is support the project financially. Both Monero and Bitcoin donations can be made to **donate.getmonero.org** if using a client that supports the [OpenAlias](https://openalias.org) standard. Alternatively, you can send XMR to the Monero donation address via the `donate` command (type `help` in the command-line wallet for details). -The Monero donation address is: `888tNkZrPN6JsEgekjMnABU4TBzc2Dt29EPAvkRxbANsAnjyPbb3iQ1YBRk1UXcdRsiKc9dhwMVgN5S9cQUiyoogDavup3H` (viewkey: `f359631075708155cc3d92a32b75a7d02a5dcf27756707b47a2b31b21c389501`) +The Monero donation address is: `888tNkZrPN6JsEgekjMnABU4TBzc2Dt29EPAvkRxbANsAnjyPbb3iQ1YBRk1UXcdRsiKc9dhwMVgN5S9cQUiyoogDavup3H` (viewkey: `f359631075708155cc3d92a32b75a7d02a5dcf27756707b47a2b31b21c389501`; base address for restoring with address and viewkey: `44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A`) The Bitcoin donation address is: `1KTexdemPdxSBcG55heUuTjDRYqbC5ZL8H` @@ -165,7 +165,7 @@ library archives (`.a`). | pkg-config | any | NO | `pkg-config` | `base-devel` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | `boost-devel` | NO | C++ libraries | | OpenSSL | basically any | NO | `libssl-dev` | `openssl` | `libressl-devel` | `openssl-devel` | NO | sha256 sum | -| libzmq | 3.0.0 | NO | `libzmq3-dev` | `zeromq` | `zeromq-devel` | `zeromq-devel` | NO | ZeroMQ library | +| libzmq | 4.2.0 | NO | `libzmq3-dev` | `zeromq` | `zeromq-devel` | `zeromq-devel` | NO | ZeroMQ library | | OpenPGM | ? | NO | `libpgm-dev` | `libpgm` | | `openpgm-devel` | NO | For ZeroMQ | | libnorm[2] | ? | NO | `libnorm-dev` | | | | YES | For ZeroMQ | | libunbound | 1.4.16 | YES | `libunbound-dev` | `unbound` | `unbound-devel` | `unbound-devel` | NO | DNS resolver | @@ -176,6 +176,7 @@ library archives (`.a`). | ldns | 1.6.17 | NO | `libldns-dev` | `ldns` | `libldns-devel` | `ldns-devel` | YES | SSL toolkit | | expat | 1.1 | NO | `libexpat1-dev` | `expat` | `expat-devel` | `expat-devel` | YES | XML parsing | | GTest | 1.5 | YES | `libgtest-dev`[1] | `gtest` | `gtest-devel` | `gtest-devel` | YES | Test suite | +| ccache | any | NO | `ccache` | `ccache` | `ccache` | `ccache` | YES | Compil. cache | | Doxygen | any | NO | `doxygen` | `doxygen` | `doxygen` | `doxygen` | YES | Documentation | | Graphviz | any | NO | `graphviz` | `graphviz` | `graphviz` | `graphviz` | YES | Documentation | | lrelease | ? | NO | `qttools5-dev-tools` | `qt5-tools` | `qt5-tools` | `qt5-linguist` | YES | Translations | @@ -183,7 +184,7 @@ library archives (`.a`). | libusb | ? | NO | `libusb-1.0-0-dev` | `libusb` | `libusb-devel` | `libusbx-devel` | YES | Hardware wallet | | libprotobuf | ? | NO | `libprotobuf-dev` | `protobuf` | `protobuf-devel` | `protobuf-devel` | YES | Hardware wallet | | protoc | ? | NO | `protobuf-compiler` | `protobuf` | `protobuf` | `protobuf-compiler` | YES | Hardware wallet | -| libudev | ? | No | `libudev-dev` | `systemd` | `eudev-libudev-devel` | `systemd-devel` | YES | Hardware wallet | +| libudev | ? | No | `libudev-dev` | `systemd` | `eudev-libudev-devel` | `systemd-devel` | YES | Hardware wallet | [1] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ``` @@ -191,7 +192,11 @@ build the library binary manually. This can be done with the following command ` Install all dependencies at once on Debian/Ubuntu: -``` sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev doxygen graphviz libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev``` +``` sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev ccache doxygen graphviz ``` + +Install all dependencies at once on openSUSE: + +``` sudo zypper ref && sudo zypper in cppzmq-devel ldns-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++``` Install all dependencies at once on macOS with the provided Brewfile: ``` brew update && brew bundle --file=contrib/brew/Brewfile ``` @@ -209,6 +214,10 @@ If you already have a repo cloned, initialize and update: `$ cd monero && git submodule init && git submodule update` +*Note*: If there are submodule differences between branches, you may need +to use ```git submodule sync && git submodule update``` after changing branches +to build successfully. + ### Build instructions Monero uses the CMake build system and a top-level [Makefile](Makefile) that @@ -270,6 +279,12 @@ Dependencies need to be built with -fPIC. Static libraries usually aren't, so yo HAVE_DOT=YES doxygen Doxyfile ``` +* **Optional**: use ccache not to rebuild translation units, that haven't really changed. Monero's CMakeLists.txt file automatically handles it + + ```bash + sudo apt install ccache + ``` + #### On the Raspberry Pi Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (2017-09-07 or later) from https://www.raspberrypi.org/downloads/raspbian/. If you are using Raspian Jessie, [please see note in the following section](#note-for-raspbian-jessie-users). @@ -300,14 +315,16 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( * Build: ```bash - make release + USE_SINGLE_BUILDDIR=1 make release ``` * Wait 4-6 hours * The resulting executables can be found in `build/release/bin` -* Add `PATH="$PATH:$HOME/monero/build/release/bin"` to `.profile` +* Add `export PATH="$PATH:$HOME/monero/build/release/bin"` to `$HOME/.profile` + +* Run `source $HOME/.profile` * Run Monero with `monerod --detach` @@ -704,14 +721,16 @@ DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip ## Pruning -As of May 2020, the full Monero blockchain file is about 80 GB. One can store a pruned blockchain, which is about 28 GB. +As of May 2020, the full Monero blockchain file is about 100 GB. One can store a pruned blockchain, which is about 30 GB. A pruned blockchain can only serve part of the historical chain data to other peers, but is otherwise identical in functionality to the full blockchain. -To use a pruned blockchain, it is best to start the initial sync with --prune-blockchain. However, it is also possible -to prune an existing blockchain using the monero-blockchain-prune tool or using the --prune-blockchain monerod option +To use a pruned blockchain, it is best to start the initial sync with `--prune-blockchain`. However, it is also possible +to prune an existing blockchain using the `monero-blockchain-prune` tool or using the `--prune-blockchain` `monerod` option with an existing chain. If an existing chain exists, pruning will temporarily require disk space to store both the full and pruned blockchains. +For more detailed information see the ['Pruning' entry in the Moneropedia](https://www.getmonero.org/resources/moneropedia/pruning.html) + ## Debugging This section contains general instructions for debugging failed installs or problems encountered with Monero. First, ensure you are running the latest version built from the Github repo. diff --git a/cmake/CheckLinkerFlag.cmake b/cmake/CheckLinkerFlag.cmake index 2b507ab71..7ecf5f610 100644 --- a/cmake/CheckLinkerFlag.cmake +++ b/cmake/CheckLinkerFlag.cmake @@ -15,6 +15,7 @@ macro(CHECK_LINKER_FLAG flag VARIABLE) ${_cle_source} COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} ${flag} CMAKE_FLAGS + "-DCMAKE_EXE_LINKER_FLAGS=${flag}" OUTPUT_VARIABLE OUTPUT) unset(_cle_source) set(CMAKE_C_FLAGS ${saved_CMAKE_C_FLAGS}) diff --git a/cmake/FindCcache.cmake b/cmake/FindCcache.cmake index a0734a312..2eb2fa2fd 100644 --- a/cmake/FindCcache.cmake +++ b/cmake/FindCcache.cmake @@ -42,12 +42,19 @@ find_program(CCACHE_FOUND ccache) if (CCACHE_FOUND) # Try to compile a test program with ccache, in order to verify if it really works. (needed on exotic setups) - # Create a temporary file with a simple program. - set(TEMP_CPP_FILE "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CMakeTmp/test-program.cpp") - file(WRITE "${TEMP_CPP_FILE}" "int main() { return 0; }") - # And run the found ccache on it. - execute_process(COMMAND "${CCACHE_FOUND}" "${CMAKE_CXX_COMPILER}" "${TEMP_CPP_FILE}" RESULT_VARIABLE RET) - if (${RET} EQUAL 0) + set(TEST_PROJECT "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CMakeTmp") + file(WRITE "${TEST_PROJECT}/CMakeLists.txt" [=[ +cmake_minimum_required(VERSION 3.5) +project(test) +option (CCACHE "") +file(WRITE "${CMAKE_SOURCE_DIR}/test.cpp" "int main() { return 0; }") +set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE}") +set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CCACHE}") +add_executable(test test.cpp) +]=]) + try_compile(RET "${TEST_PROJECT}/build" "${TEST_PROJECT}" "test" CMAKE_FLAGS -DCCACHE="${CCACHE_FOUND}") + unset(TEST_PROJECT) + if (${RET}) # Success message(STATUS "Found usable ccache: ${CCACHE_FOUND}") set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_FOUND}") diff --git a/cmake/FindLibUSB.cmake b/cmake/FindLibUSB.cmake index 79d8fba75..6944c6c45 100644 --- a/cmake/FindLibUSB.cmake +++ b/cmake/FindLibUSB.cmake @@ -134,7 +134,7 @@ if ( LibUSB_FOUND ) try_compile(LibUSB_COMPILE_TEST_PASSED ${CMAKE_BINARY_DIR} - "${CMAKE_SOURCE_DIR}/cmake/test-libusb-version.c" + "${CMAKE_CURRENT_LIST_DIR}/test-libusb-version.c" CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LibUSB_INCLUDE_DIRS}" "-DLINK_DIRECTORIES=${LibUSB_LIBRARIES}" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 511f0416e..046115bd3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -26,5 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +monero_enable_coverage() add_subdirectory(epee) diff --git a/contrib/brew/Brewfile b/contrib/brew/Brewfile index 1fdf45cfe..36b073b25 100644 --- a/contrib/brew/Brewfile +++ b/contrib/brew/Brewfile @@ -27,6 +27,7 @@ brew "miniupnpc" brew "readline" brew "ldns" brew "expat" +brew "ccache" brew "doxygen" brew "graphviz" brew "libunwind-headers" diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index 28ec972e4..0d71ddb13 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -10,8 +10,8 @@ HOST ?= $(BUILD) PATCHES_PATH = $(BASEDIR)/patches BASEDIR = $(CURDIR) HASH_LENGTH:=11 -DOWNLOAD_CONNECT_TIMEOUT:=10 -DOWNLOAD_RETRIES:=3 +DOWNLOAD_CONNECT_TIMEOUT:=30 +DOWNLOAD_RETRIES:=5 HOST_ID_SALT ?= salt BUILD_ID_SALT ?= salt diff --git a/contrib/depends/packages/boost.mk b/contrib/depends/packages/boost.mk index 0d241928e..4571d4232 100644 --- a/contrib/depends/packages/boost.mk +++ b/contrib/depends/packages/boost.mk @@ -1,6 +1,6 @@ package=boost $(package)_version=1_64_0 -$(package)_download_path=https://dl.bintray.com/boostorg/release/1.64.0/source/ +$(package)_download_path=https://downloads.sourceforge.net/project/boost/boost/1.64.0/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 $(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 $(package)_dependencies=libiconv diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 62e975e50..d80775b22 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=1.1.1i +$(package)_version=1.1.1k $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 +$(package)_sha256_hash=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/epee/demo/CMakeLists.txt b/contrib/epee/demo/CMakeLists.txt index b4ac2cc8b..d2ae0ed55 100644 --- a/contrib/epee/demo/CMakeLists.txt +++ b/contrib/epee/demo/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) set(Boost_USE_MULTITHREADED ON) #set(Boost_DEBUG 1) find_package(Boost COMPONENTS system filesystem thread date_time chrono regex ) diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h index 6b79f6d92..18d60e088 100644 --- a/contrib/epee/include/byte_slice.h +++ b/contrib/epee/include/byte_slice.h @@ -112,7 +112,7 @@ namespace epee explicit byte_slice(std::string&& buffer); //! Convert `stream` into a slice with zero allocations. - explicit byte_slice(byte_stream&& stream) noexcept; + explicit byte_slice(byte_stream&& stream, bool shrink = true); byte_slice(byte_slice&& source) noexcept; ~byte_slice() noexcept = default; diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 25f8c648b..84dc79266 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -24,211 +24,23 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include <fstream> -#include <boost/filesystem/path.hpp> -#include <boost/filesystem/operations.hpp> -#ifdef WIN32 -#include <windows.h> -#include "string_tools.h" -#endif - -// On Windows there is a problem with non-ASCII characters in path and file names -// as far as support by the standard components used is concerned: - -// The various file stream classes, e.g. std::ifstream and std::ofstream, are -// part of the GNU C++ Library / libstdc++. On the most basic level they use the -// fopen() call as defined / made accessible to programs compiled within MSYS2 -// by the stdio.h header file maintained by the MinGW project. - -// The critical point: The implementation of fopen() is part of MSVCRT, the -// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any -// Unicode support. - -// Monero code that would want to continue to use the normal file stream classes -// but WITH Unicode support could therefore not solve this problem on its own, -// but 2 different projects from 2 different maintaining groups would need changes -// in this particular direction - something probably difficult to achieve and -// with a long time to wait until all new versions / releases arrive. - -// Implemented solution approach: Circumvent the problem by stopping to use std -// file stream classes on Windows and directly use Unicode-capable WIN32 API -// calls. Most of the code doing so is concentrated in this header file here. +#include <string> +#include <ctime> namespace epee { namespace file_io_utils { - inline - bool is_file_exist(const std::string& path) - { - boost::filesystem::path p(path); - return boost::filesystem::exists(p); - } - - inline - bool save_string_to_file(const std::string& path_to_file, const std::string& str) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD bytes_written; - DWORD bytes_to_write = (DWORD)str.size(); - BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); - CloseHandle(file_handle); - if (bytes_written != bytes_to_write) - result = FALSE; - return result; -#else - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool get_file_time(const std::string& path_to_file, time_t& ft) - { - boost::system::error_code ec; - ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); - if(!ec) - return true; - else - return false; - } - - inline - bool set_file_time(const std::string& path_to_file, const time_t& ft) - { - boost::system::error_code ec; - boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); - if(!ec) - return true; - else - return false; - } - - - inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { - CloseHandle(file_handle); - return false; - } - target_str.resize(file_size); - DWORD bytes_read; - BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); - CloseHandle(file_handle); - if (bytes_read != file_size) - result = FALSE; - return result; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - - std::ifstream::pos_type file_size = fstream.tellg(); - - if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large - return false;//don't go crazy - size_t file_size_t = static_cast<size_t>(file_size); - - target_str.resize(file_size_t); - - fstream.seekg (0, std::ios::beg); - fstream.read((char*)target_str.data(), target_str.size()); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool append_string_to_file(const std::string& path_to_file, const std::string& str) - { - // No special Windows implementation because so far not used in Monero code - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } - } - - inline - bool get_file_size(const std::string& path_to_file, uint64_t &size) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER file_size; - BOOL result = GetFileSizeEx(file_handle, &file_size); - CloseHandle(file_handle); - if (result) { - size = file_size.QuadPart; - } - return size; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - size = fstream.tellg(); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - + bool is_file_exist(const std::string& path); + bool save_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_time(const std::string& path_to_file, time_t& ft); + bool set_file_time(const std::string& path_to_file, const time_t& ft); + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000); + bool append_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_size(const std::string& path_to_file, uint64_t &size); } } diff --git a/contrib/epee/include/memwipe.h b/contrib/epee/include/memwipe.h index c6e8f072c..c6f247913 100644 --- a/contrib/epee/include/memwipe.h +++ b/contrib/epee/include/memwipe.h @@ -32,6 +32,7 @@ #ifdef __cplusplus #include <array> +#include <cstddef> extern "C" { #endif diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 3be335e85..392ec11eb 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -41,7 +41,7 @@ #define MAX_LOG_FILES 50 #define MCLOG_TYPE(level, cat, color, type, x) do { \ - if (ELPP->vRegistry()->allowed(level, cat)) { \ + if (el::Loggers::allowed(level, cat)) { \ el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ } while (0) @@ -89,7 +89,7 @@ #define IFLOG(level, cat, color, type, init, x) \ do { \ - if (ELPP->vRegistry()->allowed(level, cat)) { \ + if (el::Loggers::allowed(level, cat)) { \ init; \ el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 5fffde8d5..522cdf263 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -47,7 +47,7 @@ #endif #include <iostream> -#include <boost/lexical_cast.hpp> +#include <ctime> #pragma once namespace epee @@ -115,15 +115,7 @@ namespace misc_utils } - - inline std::string get_thread_string_id() - { -#if defined(_WIN32) - return boost::lexical_cast<std::string>(GetCurrentThreadId()); -#elif defined(__GNUC__) - return boost::lexical_cast<std::string>(pthread_self()); -#endif - } + std::string get_thread_string_id(); inline bool get_gmt_time(time_t t, struct tm &tm) { diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 3c31cf22b..f40cd108a 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -265,6 +265,12 @@ namespace net_utils template<class t_callback> bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); + boost::asio::ssl::context& get_ssl_context() noexcept + { + assert(m_state != nullptr); + return m_state->ssl_context; + } + typename t_protocol_handler::config_type& get_config_object() { assert(m_state != nullptr); // always set in constructor diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index b03a03cad..89971bea2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -42,7 +42,7 @@ #include <boost/make_shared.hpp> #include <boost/thread.hpp> #include "warnings.h" -#include "string_tools.h" +#include "string_tools_lexical.h" #include "misc_language.h" #include "net/local_ip.h" #include "pragma_comp_defs.h" @@ -561,7 +561,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { // LOCK: chunking epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** - MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<(const void*)message_data); // 01234567890 // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 @@ -574,14 +574,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) while (!message.empty()) { byte_slice chunk = message.take_slice(chunksize_good); - MDEBUG("chunk_start="<<(void*)chunk.data()<<" ptr="<<message_data<<" pos="<<(chunk.data() - message_data)); + MDEBUG("chunk_start="<<(void*)chunk.data()<<" ptr="<<(const void*)message_data<<" pos="<<(chunk.data() - message_data)); MDEBUG("part of " << message.size() << ": pos="<<(chunk.data() - message_data) << " len="<<chunk.size()); bool ok = do_send_chunk(std::move(chunk)); // <====== *** all_ok = all_ok && ok; if (!all_ok) { - MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<(const void*)message_data); MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless " << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << message_size); return false; // partial failure in sending @@ -589,7 +589,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) // (in catch block, or uniq pointer) delete buf; } // each chunk - MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<(const void*)message_data); MDEBUG("do_send() m_connection_type = " << m_connection_type); diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index bf6589c92..4af4da790 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -27,14 +27,13 @@ #pragma once -#include <boost/lexical_cast.hpp> -#include <boost/regex.hpp> +#include "memwipe.h" + #include <boost/utility/string_ref.hpp> + #include <string> #include <utility> - -#include "memwipe.h" -#include "string_tools.h" +#include <list> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" @@ -66,34 +65,9 @@ namespace net_utils typedef std::list<std::pair<std::string, std::string> > fields_list; - inline - std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) - { - fields_list::const_iterator it = fields.begin(); - for(; it != fields.end(); it++) - if(!string_tools::compare_no_case(param_name, it->first)) - break; - - if(it==fields.end()) - return std::string(); + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields); - return it->second; - } - - - inline - std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) - { - std::string buff = "([\\?|&])"; - buff += param_name + "=([^&]*)"; - boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) - { - return result[2]; - } - return std::string(); - } + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri); static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 3725ef079..29ef82fb1 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -30,7 +30,6 @@ #include <ctype.h> #include <boost/shared_ptr.hpp> #include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include <boost/optional/optional.hpp> #include <boost/utility/string_ref.hpp> //#include <mbstring.h> @@ -46,6 +45,7 @@ #endif #include "string_tools.h" +#include "string_tools_lexical.h" #include "reg_exp_definer.h" #include "abstract_http_client.h" #include "http_base.h" diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index fce6d4b7e..df59a6c44 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,6 +31,7 @@ #include <cstdint> +#include "byte_stream.h" #include "net_utils_base.h" #include "span.h" @@ -83,11 +84,12 @@ namespace levin #define LEVIN_PROTOCOL_VER_0 0 #define LEVIN_PROTOCOL_VER_1 1 + template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; @@ -125,12 +127,41 @@ namespace levin } } + //! Provides space for levin (p2p) header, so that payload can be sent without copy + class message_writer + { + byte_slice finalize(uint32_t command, uint32_t flags, uint32_t return_code, bool expect_response); + public: + using header = bucket_head2; + + explicit message_writer(std::size_t reserve = 8192); + + message_writer(const message_writer&) = delete; + message_writer(message_writer&&) = default; + ~message_writer() = default; + message_writer& operator=(const message_writer&) = delete; + message_writer& operator=(message_writer&&) = default; + + //! \return Size of payload (excludes header size). + std::size_t payload_size() const noexcept + { + return buffer.size() < sizeof(header) ? 0 : buffer.size() - sizeof(header); + } + + byte_slice finalize_invoke(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, true); } + byte_slice finalize_notify(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, false); } + byte_slice finalize_response(uint32_t command, uint32_t return_code) + { + return finalize(command, LEVIN_PACKET_RESPONSE, return_code, false); + } + + //! Has space for levin header until a finalize method is used + byte_stream buffer; + }; + //! \return Intialized levin header. bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; - //! \return A levin notification message. - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); - /*! Generate a dummy levin message. \param noise_bytes Total size of the returned `byte_slice`. @@ -140,12 +171,11 @@ namespace levin /*! Generate 1+ levin messages that are identical to the noise message size. - \param noise Each levin message will be identical to the size of this - message. The bytes from this message will be used for padding. + \param noise_size Each levin message will be identical to this value. \return `nullptr` if `noise.size()` is less than the levin header size. Otherwise, a levin notification message OR 2+ levin fragment messages. Each message is `noise.size()` in length. */ - byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); + byte_slice make_fragmented_notify(const std::size_t noise_size, int command, message_writer message); } } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index f6b73a2d5..a6816cafc 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -51,6 +51,21 @@ #define MIN_BYTES_WANTED 512 #endif +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char* category) +{ + MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") + << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); +} + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "command-%u", command); + on_levin_traffic(context, initiator, sent, error, bytes, buf); +} + namespace epee { namespace levin @@ -88,11 +103,10 @@ public: uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id); template<class callback_t> - int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); @@ -122,12 +136,17 @@ class async_protocol_handler { std::string m_fragment_buffer; - bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + bool send_message(byte_slice message) { - const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); - if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + if (message.size() < sizeof(message_writer::header)) return false; + message_writer::header head; + std::memcpy(std::addressof(head), message.data(), sizeof(head)); + if(!m_pservice_endpoint->do_send(std::move(message))) + return false; + + on_levin_traffic(m_connection_context, true, true, false, head.m_cb, head.m_command); MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb << ", flags" << head.m_flags << ", r?=" << head.m_have_to_return_data @@ -523,26 +542,17 @@ public: { if(m_current_head.m_have_to_return_data) { - byte_slice return_buff; + levin::message_writer return_message{32 * 1024}; const uint32_t return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + m_current_head.m_command, buff_to_invoke, return_message.buffer, m_connection_context ); // peer_id remains unset if dropped if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) m_max_packet_size = m_config.m_max_packet_size; - bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); - head.m_return_code = SWAP32LE(return_code); - - if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) + if(!send_message(return_message.finalize_response(m_current_head.m_command, return_code))) return false; - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", flags" << head.m_flags - << ", r?=" << head.m_have_to_return_data - <<", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); @@ -619,7 +629,7 @@ public: } template<class callback_t> - bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, message_writer in_msg, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -638,7 +648,7 @@ public: if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if(!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -664,7 +674,7 @@ public: return true; } - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out) + int invoke(int command, message_writer in_msg, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -676,7 +686,7 @@ public: if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if (!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; @@ -713,25 +723,9 @@ public: return m_invoke_result_code; } - int notify(int command, const epee::span<const uint8_t> in_buff) - { - misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( - boost::bind(&async_protocol_handler::finish_outer_call, this)); - - CRITICAL_REGION_LOCAL(m_call_lock); - - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) - { - LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); - return -1; - } - - return 1; - } - - /*! Sends `message` without adding a levin header. The message must have - been created with `make_notify`, `make_noise_notify` or - `make_fragmented_notify`. See additional instructions for + /*! Sends `message` without adding a levin header. The message must have been + created with `make_noise_notify`, `make_fragmented_notify`, or + `message_writer::finalize_notify`. See additional instructions for `make_fragmented_notify`. \return 1 on success */ @@ -741,14 +735,11 @@ public: boost::bind(&async_protocol_handler::finish_outer_call, this) ); - const std::size_t length = message.size(); - if (!m_pservice_endpoint->do_send(std::move(message))) + if (!send_message(std::move(message))) { LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -769,36 +760,32 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::delete_connections(size_t count, bool incoming) { - std::vector <boost::uuids::uuid> connections; + std::vector<typename connections_map::mapped_type> connections; + + auto scope_exit_handler = misc_utils::create_scope_leave_handler([&connections]{ + for (auto &aph: connections) + aph->finish_outer_call(); + }); + CRITICAL_REGION_BEGIN(m_connects_lock); for (auto& c: m_connects) { if (c.second->m_connection_context.m_is_income == incoming) - connections.push_back(c.first); + if (c.second->start_outer_call()) + connections.push_back(c.second); } // close random connections from the provided set // TODO or better just keep removing random elements (performance) unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); shuffle(connections.begin(), connections.end(), std::default_random_engine(seed)); - while (count > 0 && connections.size() > 0) - { - try - { - auto i = connections.end() - 1; - async_protocol_handler<t_connection_context> *conn = m_connects.at(*i); - m_connects.erase(*i); - conn->close(); - connections.erase(i); - } - catch (const std::out_of_range &e) - { - MWARNING("Connection not found in m_connects, continuing"); - } - --count; - } + for (size_t i = 0; i < connections.size() && i < count; ++i) + m_connects.erase(connections[i]->get_connection_id()); CRITICAL_REGION_END(); + + for (size_t i = 0; i < connections.size() && i < count; ++i) + connections[i]->close(); } //------------------------------------------------------------------------------------------ template<class t_connection_context> @@ -842,36 +829,37 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config<t_connection_context>::invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->invoke(command, in_buff, buff_out) : r; + return LEVIN_OK == r ? aph->invoke(command, std::move(in_msg), buff_out) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> -int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config<t_connection_context>::invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->async_invoke(command, in_buff, cb, timeout) : r; + return LEVIN_OK == r ? aph->async_invoke(command, std::move(in_msg), cb, timeout) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::foreach_connection(const callback_t &cb) { - CRITICAL_REGION_LOCAL(m_connects_lock); std::vector<typename connections_map::mapped_type> conn; - conn.reserve(m_connects.size()); auto scope_exit_handler = misc_utils::create_scope_leave_handler([&conn]{ for (auto &aph: conn) aph->finish_outer_call(); }); + CRITICAL_REGION_BEGIN(m_connects_lock); + conn.reserve(m_connects.size()); for (auto &e: m_connects) if (e.second->start_outer_call()) conn.push_back(e.second); + CRITICAL_REGION_END() for (auto &aph: conn) if (!cb(aph->get_context_ref())) @@ -883,11 +871,8 @@ bool async_protocol_handler_config<t_connection_context>::foreach_connection(con template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::for_connection(const boost::uuids::uuid &connection_id, const callback_t &cb) { - CRITICAL_REGION_LOCAL(m_connects_lock); - async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); - if (!aph) - return false; - if (!aph->start_outer_call()) + async_protocol_handler<t_connection_context>* aph = nullptr; + if (find_and_lock_connection(connection_id, aph) != LEVIN_OK) return false; auto scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); @@ -935,14 +920,6 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id) -{ - async_protocol_handler<t_connection_context>* aph; - int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->notify(command, in_buff) : r; -} -//------------------------------------------------------------------------------------------ -template<class t_connection_context> int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) { async_protocol_handler<t_connection_context>* aph; @@ -953,12 +930,14 @@ int async_protocol_handler_config<t_connection_context>::send(byte_slice message template<class t_connection_context> bool async_protocol_handler_config<t_connection_context>::close(boost::uuids::uuid connection_id) { - CRITICAL_REGION_LOCAL(m_connects_lock); - async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); - if (!aph) + async_protocol_handler<t_connection_context>* aph = nullptr; + if (find_and_lock_connection(connection_id, aph) != LEVIN_OK) return false; + auto scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); if (!aph->close()) return false; + CRITICAL_REGION_LOCAL(m_connects_lock); m_connects.erase(connection_id); return true; } diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index cf637ba1d..d794dd51c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -24,12 +24,9 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - - #pragma once + #include "http_base.h" -#include "reg_exp_definer.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -38,173 +35,8 @@ namespace epee { namespace net_utils { - - inline bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) - { - enum state - { - st_param_name, - st_param_val - }; - state st = st_param_name; - std::string::const_iterator start_it = query.begin(); - std::pair<std::string, std::string> e; - for(std::string::const_iterator it = query.begin(); it != query.end(); it++) - { - switch(st) - { - case st_param_name: - if(*it == '=') - { - e.first.assign(start_it, it); - start_it = it;++start_it; - st = st_param_val; - } - break; - case st_param_val: - if(*it == '&') - { - e.second.assign(start_it, it); - start_it = it;++start_it; - params.push_back(e); - e.first.clear();e.second.clear(); - st = st_param_name; - } - break; - default: - LOG_ERROR("Unknown state " << (int)st); - return false; - } - } - if(st == st_param_name) - { - if(start_it != query.end()) - { - e.first.assign(start_it, query.end()); - params.push_back(e); - } - }else - { - if(start_it != query.end()) - e.second.assign(start_it, query.end()); - - if(e.first.size()) - params.push_back(e); - } - return true; - } - - inline - bool parse_uri(const std::string uri, http::uri_content& content) - { - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - content.m_query_params.clear(); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - - boost::smatch result; - if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); - content.m_path = uri; - return true; - } - if(result[1].matched) - { - content.m_path = result[1]; - } - if(result[3].matched) - { - content.m_query = result[3]; - } - if(result[5].matched) - { - content.m_fragment = result[5]; - } - if(content.m_query.size()) - { - parse_uri_query(content.m_query, content.m_query_params); - } - return true; - } - - inline - bool parse_url_ipv6(const std::string url_str, http::url_content& content) - { - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 3 4 5 6 7 - - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return false; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - else // if host not matched, matching should be considered failed - { - return false; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - - inline - bool parse_url(const std::string url_str, http::url_content& content) - { - - if (parse_url_ipv6(url_str, content)) return true; - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 34 5 6 7 - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return true; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - + bool parse_uri(const std::string uri, http::uri_content& content); + bool parse_url_ipv6(const std::string url_str, http::url_content& content); + bool parse_url(const std::string url_str, http::url_content& content); } } diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 1b1577e77..58cd7e45f 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -36,6 +36,7 @@ #include <boost/utility/string_ref.hpp> #include <boost/asio/ip/tcp.hpp> #include <boost/asio/ssl.hpp> +#include <boost/filesystem/path.hpp> #include <boost/system/error_code.hpp> #define SSL_FINGERPRINT_SIZE 32 @@ -144,6 +145,9 @@ namespace net_utils bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + + //! Store private key for `ssl` at `base + ".key"` unencrypted and certificate for `ssl` at `base + ".crt"`. + boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base); } } diff --git a/contrib/epee/include/reg_exp_definer.h b/contrib/epee/include/reg_exp_definer.h index eb11c9e10..386a45f9a 100644 --- a/contrib/epee/include/reg_exp_definer.h +++ b/contrib/epee/include/reg_exp_definer.h @@ -29,6 +29,7 @@ #define _REG_EXP_DEFINER_H_ #include <boost/interprocess/detail/atomic.hpp> +#include <boost/regex.hpp> #include "syncobj.h" namespace epee diff --git a/contrib/epee/include/serialization/keyvalue_serialization.h b/contrib/epee/include/serialization/keyvalue_serialization.h index fd343865c..2e4a0faad 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization.h +++ b/contrib/epee/include/serialization/keyvalue_serialization.h @@ -73,7 +73,8 @@ public: \ template<bool is_store, class t_storage> \ bool serialize_map(t_storage& stg, typename t_storage::hsection hparent_section) \ { \ - decltype(*this) &this_ref = *this; + decltype(*this) &this_ref = *this; \ + (void) this_ref; // Suppress unused var warnings. Sometimes this var is used, sometimes not. #define KV_SERIALIZE_N(varialble, val_name) \ epee::serialization::selector<is_store>::serialize(this_ref.varialble, stg, hparent_section, val_name); diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index b355c960a..26861f994 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -31,7 +31,6 @@ #include <algorithm> #include <cstdint> #include <memory> -#include <string> #include <type_traits> namespace epee @@ -167,10 +166,11 @@ namespace epee } //! make a span from a std::string - template<typename T> - span<const T> strspan(const std::string &s) noexcept + template<typename T, typename U> + span<const T> strspan(const U&s) noexcept { - static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected type"); + static_assert(std::is_same<typename U::value_type, char>(), "unexpected source type"); + static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected destination type"); return {reinterpret_cast<const T*>(s.data()), s.size()}; } } diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 802e16c1b..383d67cc2 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -37,21 +37,14 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category); + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command); + namespace { - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category) - { - MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") - << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); - } - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) - { - char buf[32]; - snprintf(buf, sizeof(buf), "command-%u", command); - return on_levin_traffic(context, initiator, sent, error, bytes, buf); - } static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { 8192, // objects 16384, // fields @@ -117,12 +110,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; + levin::message_writer to_send{16 * 1024}; std::string buff_to_recv; - stg.store_to_binary(buff_to_send, 16 * 1024); + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); + int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); @@ -145,10 +137,9 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send, 16 * 1024); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + levin::message_writer to_send{16 * 1024}; + stg.store_to_binary(to_send.buffer); + int res = transport.invoke_async(command, std::move(to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -192,11 +183,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send); + levin::message_writer to_send; + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); + int res = transport.send(to_send.finalize_notify(command), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -207,7 +197,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff, &default_levin_limits)) @@ -230,12 +220,11 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out, 32 * 1024)) + if(!strg_out.store_to_binary(buff_out)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; } - on_levin_traffic(context, false, true, false, buff_out.size(), command); return res; } @@ -262,7 +251,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -271,13 +260,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; epee::byte_slice fake_str; \ - return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ + bool handled = false; epee::byte_stream fake_str; \ + return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -297,7 +286,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index 5a6cc0b51..e59cbcf5f 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -28,13 +28,8 @@ #pragma once -#include <algorithm> -#include <boost/utility/string_ref.hpp> - -#include "misc_log_ex.h" - -#undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "serialization" +#include <boost/utility/string_ref_fwd.hpp> +#include <string> namespace epee { @@ -97,46 +92,7 @@ namespace misc_utils return lut[(uint8_t)c] & 1; } - inline std::string transform_to_escape_sequence(const std::string& src) - { - static const char escaped[] = "\b\f\n\r\t\v\"\\/"; - std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); - if (it == src.end()) - return src; - - std::string res; - res.reserve(2 * src.size()); - res.assign(src.begin(), it); - for(; it!=src.end(); ++it) - { - switch(*it) - { - case '\b': //Backspace (ascii code 08) - res+="\\b"; break; - case '\f': //Form feed (ascii code 0C) - res+="\\f"; break; - case '\n': //New line - res+="\\n"; break; - case '\r': //Carriage return - res+="\\r"; break; - case '\t': //Tab - res+="\\t"; break; - case '\v': //Vertical tab - res+="\\v"; break; - //case '\'': //Apostrophe or single quote - // res+="\\'"; break; - case '"': //Double quote - res+="\\\""; break; - case '\\': //Backslash caracter - res+="\\\\"; break; - case '/': //Backslash caracter - res+="\\/"; break; - default: - res.push_back(*it); - } - } - return res; - } + std::string transform_to_escape_sequence(const std::string& src); /* \b Backspace (ascii code 08) @@ -150,98 +106,7 @@ namespace misc_utils \\ Backslash character */ - inline void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) - { - bool escape_mode = false; - std::string::const_iterator it = star_end_string; - ++it; - std::string::const_iterator fi = it; - while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0) - ++fi; - val.assign(it, fi); - it = fi; - for(;it != buf_end;it++) - { - if(escape_mode/*prev_ch == '\\'*/) - { - switch(*it) - { - case 'b': //Backspace (ascii code 08) - val.push_back(0x08);break; - case 'f': //Form feed (ascii code 0C) - val.push_back(0x0C);break; - case 'n': //New line - val.push_back('\n');break; - case 'r': //Carriage return - val.push_back('\r');break; - case 't': //Tab - val.push_back('\t');break; - case 'v': //Vertical tab - val.push_back('\v');break; - case '\'': //Apostrophe or single quote - val.push_back('\'');break; - case '"': //Double quote - val.push_back('"');break; - case '\\': //Backslash character - val.push_back('\\');break; - case '/': //Slash character - val.push_back('/');break; - case 'u': //Unicode code point - if (buf_end - it < 4) - { - ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); - } - else - { - uint32_t dst = 0; - for (int i = 0; i < 4; ++i) - { - const unsigned char tmp = isx[(unsigned char)*++it]; - CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); - dst = dst << 4 | tmp; - } - // encode as UTF-8 - if (dst <= 0x7f) - { - val.push_back(dst); - } - else if (dst <= 0x7ff) - { - val.push_back(0xc0 | (dst >> 6)); - val.push_back(0x80 | (dst & 0x3f)); - } - else if (dst <= 0xffff) - { - val.push_back(0xe0 | (dst >> 12)); - val.push_back(0x80 | ((dst >> 6) & 0x3f)); - val.push_back(0x80 | (dst & 0x3f)); - } - else - { - ASSERT_MES_AND_THROW("Unicode code point is out or range"); - } - } - break; - default: - val.push_back(*it); - LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); - } - escape_mode = false; - }else if(*it == '"') - { - star_end_string = it; - return; - }else if(*it == '\\') - { - escape_mode = true; - } - else - { - val.push_back(*it); - } - } - ASSERT_MES_AND_THROW("Failed to match string in json entry: " << std::string(star_end_string, buf_end)); - } + void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val); inline bool match_string(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) { try @@ -255,42 +120,7 @@ namespace misc_utils return false; } } - inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val) - { - val.clear(); - uint8_t float_flag = 0; - is_signed_val = false; - size_t chars = 0; - std::string::const_iterator it = star_end_string; - if (it != buf_end && *it == '-') - { - is_signed_val = true; - ++chars; - ++it; - } - for(;it != buf_end;it++) - { - const uint8_t flags = lut[(uint8_t)*it]; - if (flags & 16) - { - float_flag |= flags; - ++chars; - } - else - { - val = boost::string_ref(&*star_end_string, chars); - if(val.size()) - { - star_end_string = --it; - is_float_val = !!(float_flag & 2); - return; - } - else - ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); - } - } - ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); - } + void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val); inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { try @@ -304,25 +134,7 @@ namespace misc_utils return false; } } - inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) - { - val.clear(); - - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) - { - if (!(lut[(uint8_t)*it] & 4)) - { - val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it)); - if(val.size()) - { - star_end_string = --it; - return; - }else - ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); - } - } - ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); - } + void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val); inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { try @@ -335,44 +147,8 @@ namespace misc_utils return false; } } - inline bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) - { - val.clear(); - - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) - { - if(!isalnum(*it) && *it != '-' && *it != '_') - { - val.assign(star_end_string, it); - if(val.size()) - { - star_end_string = --it; - return true; - }else - return false; - } - } - return false; - } - inline bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end) - { - word_end = star_end_string; - - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) - { - if(isspace(*it)) - { - - continue; - }else if( *it == '=' ) - { - star_end_string = it; - word_end = it; - return true; - } - } - return false; - } + bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val); + bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end); } } } diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index c5d0c48ee..60f3e672f 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -31,9 +31,12 @@ #include "misc_log_ex.h" #include "span.h" +#include <boost/mpl/contains.hpp> + namespace epee { class byte_slice; + class byte_stream; namespace serialization { /************************************************************************/ @@ -83,8 +86,13 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); - bool load_from_binary(const std::string& target, const limits_t *limits = NULL); + bool store_to_binary(byte_stream& ss); + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = nullptr); + bool load_from_binary(const std::string& target, const limits_t *limits = nullptr) + { + return load_from_binary(epee::strspan<uint8_t>(target), limits); + } + template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index e7250e895..7f6596f36 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -29,13 +29,15 @@ #include <string> #include "byte_slice.h" -#include "parserse_base_utils.h" +#include "parserse_base_utils.h" /// TODO: (mj-xmr) This will be reduced in an another PR #include "portable_storage.h" #include "file_io_utils.h" #include "span.h" namespace epee { + class byte_stream; + namespace serialization { //----------------------------------------------------------------------------------------------------------- @@ -127,5 +129,14 @@ namespace epee store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } + //----------------------------------------------------------------------------------------------------------- + template<class t_struct> + bool store_t_to_binary(t_struct& str_in, byte_stream& binary_buff) + { + portable_storage ps; + str_in.store(ps); + return ps.store_to_binary(binary_buff); + } + } } diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 2d9317d60..dbbe1906e 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -24,33 +24,16 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #ifndef _STRING_TOOLS_H_ #define _STRING_TOOLS_H_ -// Previously pulled in by ASIO, further cleanup still required ... -#ifdef _WIN32 -# include <winsock2.h> -# include <windows.h> -#endif - -#include <string.h> -#include <locale> -#include <cstdlib> -#include <string> -#include <type_traits> -#include <boost/lexical_cast.hpp> -#include <boost/algorithm/string/predicate.hpp> -#include <boost/utility/string_ref.hpp> -#include "misc_log_ex.h" -#include "storages/parserse_base_utils.h" #include "hex.h" -#include "memwipe.h" #include "mlocker.h" -#include "span.h" -#include "warnings.h" +#include <boost/utility/string_ref.hpp> +#include <sstream> +#include <string> +#include <cstdint> #ifndef OUT #define OUT @@ -74,202 +57,38 @@ namespace string_tools { return from_hex::to_string(res, s); } - //---------------------------------------------------------------------------- -PUSH_WARNINGS -DISABLE_GCC_WARNING(maybe-uninitialized) - template<class XType> - inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) - { - if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) - { - for (char c : str_id) - { - if (!epee::misc_utils::parse::isdigit(c)) - return false; - } - } - - try - { - val = boost::lexical_cast<XType>(str_id); - return true; - } - catch(const std::exception& /*e*/) - { - //const char* pmsg = e.what(); - return false; - } - catch(...) - { - return false; - } - - return true; - } -POP_WARNINGS - //---------------------------------------------------------------------------- - template<class XType> - inline bool xtype_to_string(const XType& val, std::string& str) - { - try - { - str = boost::lexical_cast<std::string>(val); - } - catch(...) - { - return false; - } - - return true; - } - //---------------------------------------------------------------------------- - std::string get_ip_string_from_int32(uint32_t ip); - //---------------------------------------------------------------------------- - bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); - //---------------------------------------------------------------------------- - inline bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) - { - //parse ip and address - std::string::size_type p = addres.find(':'); - std::string ip_str, port_str; - if(p == std::string::npos) - { - port = 0; - ip_str = addres; - } - else - { - ip_str = addres.substr(0, p); - port_str = addres.substr(p+1, addres.size()); - } - - if(!get_ip_int32_from_string(ip, ip_str)) - { - return false; - } - - if(p != std::string::npos && !get_xtype_from_string(port, port_str)) - { - return false; - } - return true; - } - - inline std::string num_to_string_fast(int64_t val) - { - /* - char buff[30] = {0}; - i64toa_s(val, buff, sizeof(buff)-1, 10); - return buff;*/ - return boost::lexical_cast<std::string>(val); - } - //---------------------------------------------------------------------------- - template<typename T> - inline std::string to_string_hex(const T &val) - { - static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); - std::stringstream ss; - ss << std::hex << val; - std::string s; - ss >> s; - return s; - } - //---------------------------------------------------------------------------- + + std::string get_ip_string_from_int32(uint32_t ip); + bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres); + std::string num_to_string_fast(int64_t val); - inline bool compare_no_case(const std::string& str1, const std::string& str2) - { - - return !boost::iequals(str1, str2); - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_name() - { - static std::string module_name; - return module_name; - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_folder() - { - static std::string module_folder; - return module_folder; - } - //---------------------------------------------------------------------------- + bool compare_no_case(const std::string& str1, const std::string& str2); + std::string& get_current_module_name(); + std::string& get_current_module_folder(); #ifdef _WIN32 - inline std::string get_current_module_path() - { - char pname [5000] = {0}; - GetModuleFileNameA( NULL, pname, sizeof(pname)); - pname[sizeof(pname)-1] = 0; //be happy ;) - return pname; - } + std::string get_current_module_path(); #endif - //---------------------------------------------------------------------------- - inline bool set_module_name_and_folder(const std::string& path_to_process_) - { - std::string path_to_process = path_to_process_; -#ifdef _WIN32 - path_to_process = get_current_module_path(); -#endif - std::string::size_type a = path_to_process.rfind( '\\' ); - if(a == std::string::npos ) - { - a = path_to_process.rfind( '/' ); - } - if ( a != std::string::npos ) - { - get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); - get_current_module_folder() = path_to_process.substr(0, a); - return true; - }else - return false; - - } - - //---------------------------------------------------------------------------- - inline bool trim_left(std::string& str) - { - for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) - str.erase(str.begin()); - - return true; - } - //---------------------------------------------------------------------------- - inline bool trim_right(std::string& str) - { - - for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) - str.erase( --((it++).base())); - - return true; - } - //---------------------------------------------------------------------------- - inline std::string& trim(std::string& str) - { - - trim_left(str); - trim_right(str); - return str; - } + bool set_module_name_and_folder(const std::string& path_to_process_); + bool trim_left(std::string& str); + bool trim_right(std::string& str); //---------------------------------------------------------------------------- - inline std::string trim(const std::string& str_) + inline std::string& trim(std::string& str) { - std::string str = str_; trim_left(str); trim_right(str); return str; } //---------------------------------------------------------------------------- - inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false) + inline std::string trim(const std::string& str_) { - if (s.size() < n) - { - if (prepend) - s = std::string(n - s.size(), c) + s; - else - s.append(n - s.size(), c); - } - return s; + std::string str = str_; + trim_left(str); + trim_right(str); + return str; } + std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false); + //---------------------------------------------------------------------------- template<class t_pod_type> std::string pod_to_hex(const t_pod_type& s) @@ -296,64 +115,25 @@ POP_WARNINGS { return hex_to_pod(hex_str, unwrap(s)); } - //---------------------------------------------------------------------------- - bool validate_hex(uint64_t length, const std::string& str); - //---------------------------------------------------------------------------- - inline std::string get_extension(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return res; - - res = str.substr(pos+1, str.size()-pos); - return res; - } - //---------------------------------------------------------------------------- - inline std::string cut_off_extension(const std::string& str) + //---------------------------------------------------------------------------- + template<typename T> + inline std::string to_string_hex(const T &val) { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return str; - - res = str.substr(0, pos); - return res; + static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); + std::stringstream ss; + ss << std::hex << val; + std::string s; + ss >> s; + return s; } - //---------------------------------------------------------------------------- + + bool validate_hex(uint64_t length, const std::string& str); + std::string get_extension(const std::string& str); + std::string cut_off_extension(const std::string& str); + #ifdef _WIN32 - inline std::wstring utf8_to_utf16(const std::string& str) - { - if (str.empty()) - return {}; - int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); - if (wstr_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::wstring wstr(wstr_size, wchar_t{}); - if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return wstr; - } - inline std::string utf16_to_utf8(const std::wstring& wstr) - { - if (wstr.empty()) - return {}; - int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); - if (str_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::string str(str_size, char{}); - if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return str; - } + std::wstring utf8_to_utf16(const std::string& str); + std::string utf16_to_utf8(const std::wstring& wstr); #endif } } diff --git a/contrib/epee/include/string_tools_lexical.h b/contrib/epee/include/string_tools_lexical.h new file mode 100644 index 000000000..01be562bc --- /dev/null +++ b/contrib/epee/include/string_tools_lexical.h @@ -0,0 +1,91 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef _STRING_TOOLS_LEXICAL_H_ +#define _STRING_TOOLS_LEXICAL_H_ + +#include "warnings.h" +#include "storages/parserse_base_utils.h" +#include <boost/lexical_cast.hpp> // A heavy header, that was extracted from the rest + +#ifndef OUT + #define OUT +#endif + +namespace epee +{ +namespace string_tools +{ +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) + template<class XType> + inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) + { + if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) + { + for (char c : str_id) + { + if (!epee::misc_utils::parse::isdigit(c)) + return false; + } + } + + try + { + val = boost::lexical_cast<XType>(str_id); + return true; + } + catch(const std::exception& /*e*/) + { + //const char* pmsg = e.what(); + return false; + } + catch(...) + { + return false; + } + + return true; + } +POP_WARNINGS + + template<class XType> + inline bool xtype_to_string(const XType& val, std::string& str) + { + try + { + str = boost::lexical_cast<std::string>(val); + } + catch(...) + { + return false; + } + + return true; + } +} +} +#endif //_STRING_TOOLS_LEXICAL_H_ diff --git a/contrib/epee/include/tiny_ini.h b/contrib/epee/include/tiny_ini.h index 2bc71fc1a..6ced548eb 100644 --- a/contrib/epee/include/tiny_ini.h +++ b/contrib/epee/include/tiny_ini.h @@ -28,8 +28,6 @@ #ifndef _TINY_INI_H_ #define _TINY_INI_H_ -#include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include "string_tools.h" namespace epee @@ -37,20 +35,8 @@ namespace epee namespace tiny_ini { - inline - bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) - { - std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; - const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) - return false; - res = result[2]; - string_tools::trim(res); - return true; - } - inline - std::string get_param_value(const std::string& param_name, const std::string& ini_entry) + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res); + inline std::string get_param_value(const std::string& param_name, const std::string& ini_entry) { std::string buff; get_param_value(param_name, ini_entry, buff); diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 641d4718e..3c5eb49e9 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,15 +26,25 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set(EPEE_INCLUDE_DIR_BASE "${CMAKE_CURRENT_SOURCE_DIR}/../include") -add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp +# Add headers to the file list, to be able to search for them and autosave in IDEs. +monero_find_all_headers(EPEE_HEADERS_PUBLIC "${EPEE_INCLUDE_DIR_BASE}") + +monero_add_library(epee byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp parserse_base_utils.cpp wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp int-util.cpp portable_storage.cpp misc_language.cpp + misc_os_dependent.cpp + file_io_utils.cpp + net_parse_helpers.cpp + http_base.cpp + tiny_ini.cpp + ${EPEE_HEADERS_PUBLIC} ) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) - add_library(epee_readline STATIC readline_buffer.cpp) + monero_add_library(epee_readline readline_buffer.cpp) endif() if(HAVE_C11) @@ -62,8 +72,9 @@ target_link_libraries(epee ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - PRIVATE + ${Boost_REGEX_LIBRARY} ${OPENSSL_LIBRARIES} + PRIVATE ${EXTRA_LIBRARIES}) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) @@ -74,5 +85,8 @@ if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) ${GNU_READLINE_LIBRARY}) endif() -target_include_directories(epee PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include") +target_include_directories(epee + PUBLIC + "${EPEE_INCLUDE_DIR_BASE}" + "${OPENSSL_INCLUDE_DIR}") diff --git a/contrib/epee/src/abstract_http_client.cpp b/contrib/epee/src/abstract_http_client.cpp index 540917873..3ae09c90e 100644 --- a/contrib/epee/src/abstract_http_client.cpp +++ b/contrib/epee/src/abstract_http_client.cpp @@ -1,6 +1,7 @@ #include "net/abstract_http_client.h" #include "net/http_base.h" #include "net/net_parse_helpers.h" +#include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp index faf7689be..430853c64 100644 --- a/contrib/epee/src/byte_slice.cpp +++ b/contrib/epee/src/byte_slice.cpp @@ -36,6 +36,11 @@ #include "byte_slice.h" #include "byte_stream.h" +namespace +{ + const std::size_t page_size = 4096; +} + namespace epee { struct byte_slice_data @@ -146,7 +151,7 @@ namespace epee : byte_slice() { std::size_t space_needed = 0; - for (const auto source : sources) + for (const auto& source : sources) space_needed += source.size(); if (space_needed) @@ -155,7 +160,7 @@ namespace epee span<std::uint8_t> out{reinterpret_cast<std::uint8_t*>(storage.get() + 1), space_needed}; portion_ = {out.data(), out.size()}; - for (const auto source : sources) + for (const auto& source : sources) { std::memcpy(out.data(), source.data(), source.size()); if (out.remove_prefix(source.size()) < source.size()) @@ -173,16 +178,27 @@ namespace epee : byte_slice(adapt_buffer{}, std::move(buffer)) {} - byte_slice::byte_slice(byte_stream&& stream) noexcept + byte_slice::byte_slice(byte_stream&& stream, const bool shrink) : storage_(nullptr), portion_(stream.data(), stream.size()) { - if (stream.size()) + if (portion_.size()) { - std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice); + byte_buffer buf; + if (shrink && page_size <= stream.available()) + { + buf = byte_buffer_resize(stream.take_buffer(), portion_.size()); + if (!buf) + throw std::bad_alloc{}; + portion_ = {buf.get(), portion_.size()}; + } + else // no need to shrink buffer + buf = stream.take_buffer(); + + std::uint8_t* const data = buf.release() - sizeof(raw_byte_slice); new (data) raw_byte_slice{}; storage_.reset(reinterpret_cast<raw_byte_slice*>(data)); } - else + else // empty stream portion_ = nullptr; } diff --git a/contrib/epee/src/file_io_utils.cpp b/contrib/epee/src/file_io_utils.cpp new file mode 100644 index 000000000..5072adcbc --- /dev/null +++ b/contrib/epee/src/file_io_utils.cpp @@ -0,0 +1,231 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "file_io_utils.h" + +#include <fstream> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> +#ifdef WIN32 +#include <windows.h> +#include "string_tools.h" +#endif + +// On Windows there is a problem with non-ASCII characters in path and file names +// as far as support by the standard components used is concerned: + +// The various file stream classes, e.g. std::ifstream and std::ofstream, are +// part of the GNU C++ Library / libstdc++. On the most basic level they use the +// fopen() call as defined / made accessible to programs compiled within MSYS2 +// by the stdio.h header file maintained by the MinGW project. + +// The critical point: The implementation of fopen() is part of MSVCRT, the +// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any +// Unicode support. + +// Monero code that would want to continue to use the normal file stream classes +// but WITH Unicode support could therefore not solve this problem on its own, +// but 2 different projects from 2 different maintaining groups would need changes +// in this particular direction - something probably difficult to achieve and +// with a long time to wait until all new versions / releases arrive. + +// Implemented solution approach: Circumvent the problem by stopping to use std +// file stream classes on Windows and directly use Unicode-capable WIN32 API +// calls. Most of the code doing so is concentrated in this header file here. + +namespace epee +{ +namespace file_io_utils +{ + + bool is_file_exist(const std::string& path) + { + boost::filesystem::path p(path); + return boost::filesystem::exists(p); + } + + + bool save_string_to_file(const std::string& path_to_file, const std::string& str) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD bytes_written; + DWORD bytes_to_write = (DWORD)str.size(); + BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); + CloseHandle(file_handle); + if (bytes_written != bytes_to_write) + result = FALSE; + return result; +#else + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool get_file_time(const std::string& path_to_file, time_t& ft) + { + boost::system::error_code ec; + ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); + if(!ec) + return true; + else + return false; + } + + + bool set_file_time(const std::string& path_to_file, const time_t& ft) + { + boost::system::error_code ec; + boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); + if(!ec) + return true; + else + return false; + } + + + + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD file_size = GetFileSize(file_handle, NULL); + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { + CloseHandle(file_handle); + return false; + } + target_str.resize(file_size); + DWORD bytes_read; + BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); + CloseHandle(file_handle); + if (bytes_read != file_size) + result = FALSE; + return result; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + + std::ifstream::pos_type file_size = fstream.tellg(); + + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large + return false;//don't go crazy + size_t file_size_t = static_cast<size_t>(file_size); + + target_str.resize(file_size_t); + + fstream.seekg (0, std::ios::beg); + fstream.read((char*)target_str.data(), target_str.size()); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool append_string_to_file(const std::string& path_to_file, const std::string& str) + { + // No special Windows implementation because so far not used in Monero code + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } + } + + + bool get_file_size(const std::string& path_to_file, uint64_t &size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER file_size; + BOOL result = GetFileSizeEx(file_handle, &file_size); + CloseHandle(file_handle); + if (result) { + size = file_size.QuadPart; + } + return size; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + size = fstream.tellg(); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + +} +} diff --git a/contrib/epee/src/http_base.cpp b/contrib/epee/src/http_base.cpp new file mode 100644 index 000000000..647dfb899 --- /dev/null +++ b/contrib/epee/src/http_base.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/http_base.h" +#include "memwipe.h" +#include "string_tools.h" + +#include <boost/regex.hpp> +#include <string> +#include <utility> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.http" + +namespace epee +{ +namespace net_utils +{ +namespace http +{ + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) + { + fields_list::const_iterator it = fields.begin(); + for(; it != fields.end(); it++) + if(!string_tools::compare_no_case(param_name, it->first)) + break; + + if(it==fields.end()) + return std::string(); + + return it->second; + } + + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) + { + std::string buff = "([\\?|&])"; + buff += param_name + "=([^&]*)"; + boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) + { + return result[2]; + } + return std::string(); + } +} +} +} diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp index 5ec86b3d6..7c5cd5a78 100644 --- a/contrib/epee/src/levin_base.cpp +++ b/contrib/epee/src/levin_base.cpp @@ -34,6 +34,25 @@ namespace epee { namespace levin { + message_writer::message_writer(const std::size_t reserve) + : buffer() + { + buffer.reserve(reserve); + buffer.put_n(0, sizeof(header)); + } + + byte_slice message_writer::finalize(const uint32_t command, const uint32_t flags, const uint32_t return_code, const bool expect_response) + { + if (buffer.size() < sizeof(header)) + throw std::runtime_error{"levin_writer::finalize already called"}; + + header head = make_header(command, payload_size(), flags, expect_response); + head.m_return_code = SWAP32LE(return_code); + + std::memcpy(buffer.tellp() - buffer.size(), std::addressof(head), sizeof(head)); + return byte_slice{std::move(buffer)}; + } + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept { bucket_head2 head = {0}; @@ -47,12 +66,6 @@ namespace levin return head; } - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) - { - const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); - return byte_slice{epee::as_byte_span(head), payload}; - } - byte_slice make_noise_notify(const std::size_t noise_bytes) { static constexpr const std::uint32_t flags = @@ -68,46 +81,40 @@ namespace levin return byte_slice{std::move(buffer)}; } - byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + byte_slice make_fragmented_notify(const std::size_t noise_size, const int command, message_writer message) { - const size_t noise_size = noise_message.size(); if (noise_size < sizeof(bucket_head2) * 2) return nullptr; - if (payload.size() <= noise_size - sizeof(bucket_head2)) + if (message.buffer.size() <= noise_size) { /* The entire message can be sent at once, and the levin binary parser will ignore extra bytes. So just pad with zeroes and otherwise send a "normal", not fragmented message. */ - const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); - const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; - const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); - return byte_slice{as_byte_span(head), payload, padding_bytes}; + message.buffer.put_n(0, noise_size - message.buffer.size()); + return message.finalize_notify(command); } // fragment message + const byte_slice payload_bytes = message.finalize_notify(command); + span<const std::uint8_t> payload = to_span(payload_bytes); + const size_t payload_space = noise_size - sizeof(bucket_head2); const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; - std::string buffer{}; - buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + byte_stream buffer{}; + buffer.reserve(expected_fragments * noise_size); - bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + bucket_head2 head = make_header(0, payload_space, LEVIN_PACKET_BEGIN, false); + buffer.write(as_byte_span(head)); - head.m_command = command; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_cb = payload.size(); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + // internal levin header is in payload already - size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + size_t copy_size = payload.remove_prefix(payload_space); + buffer.write(payload.data() - copy_size, copy_size); - head.m_command = 0; head.m_flags = 0; - head.m_cb = noise_size - sizeof(bucket_head2); - while (!payload.empty()) { copy_size = payload.remove_prefix(payload_space); @@ -115,12 +122,12 @@ namespace levin if (payload.empty()) head.m_flags = LEVIN_PACKET_END; - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + buffer.write(as_byte_span(head)); + buffer.write(payload.data() - copy_size, copy_size); } const size_t padding = noise_size - copy_size - sizeof(bucket_head2); - buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + buffer.put_n(0, padding); return byte_slice{std::move(buffer)}; } diff --git a/contrib/epee/src/misc_os_dependent.cpp b/contrib/epee/src/misc_os_dependent.cpp new file mode 100644 index 000000000..cd4967131 --- /dev/null +++ b/contrib/epee/src/misc_os_dependent.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_os_dependent.h" +#include <boost/lexical_cast.hpp> + +namespace epee +{ +namespace misc_utils +{ + // TODO: (vtnerd) This function is weird since boost::this_thread::get_id() exists but returns a different value. + std::string get_thread_string_id() + { +#if defined(_WIN32) + return boost::lexical_cast<std::string>(GetCurrentThreadId()); +#elif defined(__GNUC__) + return boost::lexical_cast<std::string>(pthread_self()); +#endif + } +} +} diff --git a/contrib/epee/src/net_parse_helpers.cpp b/contrib/epee/src/net_parse_helpers.cpp new file mode 100644 index 000000000..de7843b67 --- /dev/null +++ b/contrib/epee/src/net_parse_helpers.cpp @@ -0,0 +1,206 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/net_parse_helpers.h" +#include "net/http_base.h" +#include "misc_log_ex.h" +#include "reg_exp_definer.h" +#include <boost/lexical_cast.hpp> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net" + +namespace epee +{ +namespace net_utils +{ + + static bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) + { + enum state + { + st_param_name, + st_param_val + }; + state st = st_param_name; + std::string::const_iterator start_it = query.begin(); + std::pair<std::string, std::string> e; + for(std::string::const_iterator it = query.begin(); it != query.end(); it++) + { + switch(st) + { + case st_param_name: + if(*it == '=') + { + e.first.assign(start_it, it); + start_it = it;++start_it; + st = st_param_val; + } + break; + case st_param_val: + if(*it == '&') + { + e.second.assign(start_it, it); + start_it = it;++start_it; + params.push_back(e); + e.first.clear();e.second.clear(); + st = st_param_name; + } + break; + default: + LOG_ERROR("Unknown state " << (int)st); + return false; + } + } + if(st == st_param_name) + { + if(start_it != query.end()) + { + e.first.assign(start_it, query.end()); + params.push_back(e); + } + }else + { + if(start_it != query.end()) + e.second.assign(start_it, query.end()); + + if(e.first.size()) + params.push_back(e); + } + return true; + } + + bool parse_uri(const std::string uri, http::uri_content& content) + { + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + content.m_query_params.clear(); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + + boost::smatch result; + if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); + content.m_path = uri; + return true; + } + if(result[1].matched) + { + content.m_path = result[1]; + } + if(result[3].matched) + { + content.m_query = result[3]; + } + if(result[5].matched) + { + content.m_fragment = result[5]; + } + if(content.m_query.size()) + { + parse_uri_query(content.m_query, content.m_query_params); + } + return true; + } + + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 + + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + bool parse_url(const std::string url_str, http::url_content& content) + { + + if (parse_url_ipv6(url_str, content)) return true; + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 34 5 6 7 + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return true; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + +} +} diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index 6ed27efa9..765dadce3 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -29,6 +29,8 @@ #include <string.h> #include <thread> #include <boost/asio/ssl.hpp> +#include <boost/cerrno.hpp> +#include <boost/filesystem/operations.hpp> #include <boost/lambda/lambda.hpp> #include <openssl/ssl.h> #include <openssl/pem.h> @@ -567,6 +569,51 @@ bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s) return true; } +boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base) +{ + EVP_PKEY* ssl_key = nullptr; + X509* ssl_cert = nullptr; + const auto ctx = ssl.native_handle(); + CHECK_AND_ASSERT_MES(ctx, boost::system::error_code(EINVAL, boost::system::system_category()), "Context is null"); + CHECK_AND_ASSERT_MES(base.has_filename(), boost::system::error_code(EINVAL, boost::system::system_category()), "Need filename"); + if (!(ssl_key = SSL_CTX_get0_privatekey(ctx)) || !(ssl_cert = SSL_CTX_get0_certificate(ctx))) + return {EINVAL, boost::system::system_category()}; + + using file_closer = int(std::FILE*); + boost::system::error_code error{}; + std::unique_ptr<std::FILE, file_closer*> file{nullptr, std::fclose}; + + // write key file unencrypted + { + const boost::filesystem::path key_file{base.string() + ".key"}; + file.reset(std::fopen(key_file.string().c_str(), "wb")); + if (!file) + return {errno, boost::system::system_category()}; + boost::filesystem::permissions(key_file, boost::filesystem::owner_read, error); + if (error) + return error; + if (!PEM_write_PrivateKey(file.get(), ssl_key, nullptr, nullptr, 0, nullptr, nullptr)) + return boost::asio::error::ssl_errors(ERR_get_error()); + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + } + + // write certificate file in standard SSL X.509 unencrypted + const boost::filesystem::path cert_file{base.string() + ".crt"}; + file.reset(std::fopen(cert_file.string().c_str(), "wb")); + if (!file) + return {errno, boost::system::system_category()}; + const auto cert_perms = (boost::filesystem::owner_read | boost::filesystem::group_read | boost::filesystem::others_read); + boost::filesystem::permissions(cert_file, cert_perms, error); + if (error) + return error; + if (!PEM_write_X509(file.get(), ssl_cert)) + return boost::asio::error::ssl_errors(ERR_get_error()); + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + return error; +} + } // namespace } // namespace diff --git a/contrib/epee/src/parserse_base_utils.cpp b/contrib/epee/src/parserse_base_utils.cpp new file mode 100644 index 000000000..112e9c5e4 --- /dev/null +++ b/contrib/epee/src/parserse_base_utils.cpp @@ -0,0 +1,282 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "storages/parserse_base_utils.h" + +#include "misc_log_ex.h" +#include <boost/utility/string_ref.hpp> +#include <algorithm> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + +namespace epee +{ +namespace misc_utils +{ + namespace parse + { + std::string transform_to_escape_sequence(const std::string& src) + { + static const char escaped[] = "\b\f\n\r\t\v\"\\/"; + std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); + if (it == src.end()) + return src; + + std::string res; + res.reserve(2 * src.size()); + res.assign(src.begin(), it); + for(; it!=src.end(); ++it) + { + switch(*it) + { + case '\b': //Backspace (ascii code 08) + res+="\\b"; break; + case '\f': //Form feed (ascii code 0C) + res+="\\f"; break; + case '\n': //New line + res+="\\n"; break; + case '\r': //Carriage return + res+="\\r"; break; + case '\t': //Tab + res+="\\t"; break; + case '\v': //Vertical tab + res+="\\v"; break; + //case '\'': //Apostrophe or single quote + // res+="\\'"; break; + case '"': //Double quote + res+="\\\""; break; + case '\\': //Backslash caracter + res+="\\\\"; break; + case '/': //Backslash caracter + res+="\\/"; break; + default: + res.push_back(*it); + } + } + return res; + } + /* + + \b Backspace (ascii code 08) + \f Form feed (ascii code 0C) + \n New line + \r Carriage return + \t Tab + \v Vertical tab + \' Apostrophe or single quote + \" Double quote + \\ Backslash character + + */ + void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + { + bool escape_mode = false; + std::string::const_iterator it = star_end_string; + ++it; + std::string::const_iterator fi = it; + while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0) + ++fi; + val.assign(it, fi); + it = fi; + for(;it != buf_end;it++) + { + if(escape_mode/*prev_ch == '\\'*/) + { + switch(*it) + { + case 'b': //Backspace (ascii code 08) + val.push_back(0x08);break; + case 'f': //Form feed (ascii code 0C) + val.push_back(0x0C);break; + case 'n': //New line + val.push_back('\n');break; + case 'r': //Carriage return + val.push_back('\r');break; + case 't': //Tab + val.push_back('\t');break; + case 'v': //Vertical tab + val.push_back('\v');break; + case '\'': //Apostrophe or single quote + val.push_back('\'');break; + case '"': //Double quote + val.push_back('"');break; + case '\\': //Backslash character + val.push_back('\\');break; + case '/': //Slash character + val.push_back('/');break; + case 'u': //Unicode code point + if (buf_end - it < 4) + { + ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); + } + else + { + uint32_t dst = 0; + for (int i = 0; i < 4; ++i) + { + const unsigned char tmp = isx[(unsigned char)*++it]; + CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); + dst = dst << 4 | tmp; + } + // encode as UTF-8 + if (dst <= 0x7f) + { + val.push_back(dst); + } + else if (dst <= 0x7ff) + { + val.push_back(0xc0 | (dst >> 6)); + val.push_back(0x80 | (dst & 0x3f)); + } + else if (dst <= 0xffff) + { + val.push_back(0xe0 | (dst >> 12)); + val.push_back(0x80 | ((dst >> 6) & 0x3f)); + val.push_back(0x80 | (dst & 0x3f)); + } + else + { + ASSERT_MES_AND_THROW("Unicode code point is out or range"); + } + } + break; + default: + val.push_back(*it); + LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); + } + escape_mode = false; + }else if(*it == '"') + { + star_end_string = it; + return; + }else if(*it == '\\') + { + escape_mode = true; + } + else + { + val.push_back(*it); + } + } + ASSERT_MES_AND_THROW("Failed to match string in json entry: " << std::string(star_end_string, buf_end)); + } + void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val) + { + val.clear(); + uint8_t float_flag = 0; + is_signed_val = false; + size_t chars = 0; + std::string::const_iterator it = star_end_string; + if (it != buf_end && *it == '-') + { + is_signed_val = true; + ++chars; + ++it; + } + for(;it != buf_end;it++) + { + const uint8_t flags = lut[(uint8_t)*it]; + if (flags & 16) + { + float_flag |= flags; + ++chars; + } + else + { + val = boost::string_ref(&*star_end_string, chars); + if(val.size()) + { + star_end_string = --it; + is_float_val = !!(float_flag & 2); + return; + } + else + ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); + } + } + ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); + } + void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) + { + val.clear(); + + for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + { + if (!(lut[(uint8_t)*it] & 4)) + { + val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it)); + if(val.size()) + { + star_end_string = --it; + return; + }else + ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); + } + } + ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); + } + bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + { + val.clear(); + + for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + { + if(!isalnum(*it) && *it != '-' && *it != '_') + { + val.assign(star_end_string, it); + if(val.size()) + { + star_end_string = --it; + return true; + }else + return false; + } + } + return false; + } + bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end) + { + word_end = star_end_string; + + for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + { + if(isspace(*it)) + { + + continue; + }else if( *it == '=' ) + { + star_end_string = it; + word_end = it; + return true; + } + } + return false; + } + } +} +} diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index c3c9ccc02..b922cc9e3 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -48,15 +48,23 @@ namespace serialization TRY_ENTRY(); byte_stream ss; ss.reserve(initial_buffer_size); + store_to_binary(ss); + target = epee::byte_slice{std::move(ss)}; + return true; + CATCH_ENTRY("portable_storage::store_to_binary", false); + } + + bool portable_storage::store_to_binary(byte_stream& ss) + { + TRY_ENTRY(); storage_block_header sbh{}; sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; ss.write(epee::as_byte_span(sbh)); pack_entry_to_buff(ss, m_root); - target = epee::byte_slice{std::move(ss)}; return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) + CATCH_ENTRY("portable_storage::store_to_binary", false); } bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) @@ -76,11 +84,6 @@ namespace serialization CATCH_ENTRY("portable_storage::load_from_json", false) } - bool portable_storage::load_from_binary(const std::string& target, const limits_t *limits) - { - return load_from_binary(epee::strspan<uint8_t>(target), limits); - } - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) { m_root.m_entries.clear(); diff --git a/contrib/epee/src/string_tools.cpp b/contrib/epee/src/string_tools.cpp index fd0254016..984a151b5 100644 --- a/contrib/epee/src/string_tools.cpp +++ b/contrib/epee/src/string_tools.cpp @@ -25,6 +25,29 @@ // #include "string_tools.h" +#include "string_tools_lexical.h" + + +// Previously pulled in by ASIO, further cleanup still required ... +#ifdef _WIN32 +# include <winsock2.h> +# include <windows.h> +#endif + +#include <locale> +#include <cstdlib> +#include <string> +#include <type_traits> +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/utility/string_ref.hpp> +#include "misc_log_ex.h" +#include "storages/parserse_base_utils.h" +#include "hex.h" +#include "memwipe.h" +#include "mlocker.h" +#include "span.h" +#include "warnings.h" #include <ctype.h> @@ -68,6 +91,180 @@ namespace string_tools return false; return true; } + //---------------------------------------------------------------------------- + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) + { + //parse ip and address + std::string::size_type p = addres.find(':'); + std::string ip_str, port_str; + if(p == std::string::npos) + { + port = 0; + ip_str = addres; + } + else + { + ip_str = addres.substr(0, p); + port_str = addres.substr(p+1, addres.size()); + } + + if(!get_ip_int32_from_string(ip, ip_str)) + { + return false; + } + + if(p != std::string::npos && !get_xtype_from_string(port, port_str)) + { + return false; + } + return true; + } + + std::string num_to_string_fast(int64_t val) + { + /* + char buff[30] = {0}; + i64toa_s(val, buff, sizeof(buff)-1, 10); + return buff;*/ + return boost::lexical_cast<std::string>(val); + } + + + bool compare_no_case(const std::string& str1, const std::string& str2) + { + + return !boost::iequals(str1, str2); + } + //---------------------------------------------------------------------------- + std::string& get_current_module_name() + { + static std::string module_name; + return module_name; + } + //---------------------------------------------------------------------------- + std::string& get_current_module_folder() + { + static std::string module_folder; + return module_folder; + } + +#ifdef _WIN32 + std::string get_current_module_path() + { + char pname [5000] = {0}; + GetModuleFileNameA( NULL, pname, sizeof(pname)); + pname[sizeof(pname)-1] = 0; //be happy ;) + return pname; + } +#endif + + bool set_module_name_and_folder(const std::string& path_to_process_) + { + std::string path_to_process = path_to_process_; +#ifdef _WIN32 + path_to_process = get_current_module_path(); +#endif + std::string::size_type a = path_to_process.rfind( '\\' ); + if(a == std::string::npos ) + { + a = path_to_process.rfind( '/' ); + } + if ( a != std::string::npos ) + { + get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); + get_current_module_folder() = path_to_process.substr(0, a); + return true; + }else + return false; + + } + + //---------------------------------------------------------------------------- + bool trim_left(std::string& str) + { + for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) + str.erase(str.begin()); + + return true; + } + //---------------------------------------------------------------------------- + bool trim_right(std::string& str) + { + + for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) + str.erase( --((it++).base())); + + return true; + } + //---------------------------------------------------------------------------- + std::string pad_string(std::string s, size_t n, char c, bool prepend) + { + if (s.size() < n) + { + if (prepend) + s = std::string(n - s.size(), c) + s; + else + s.append(n - s.size(), c); + } + return s; + } + + std::string get_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return res; + + res = str.substr(pos+1, str.size()-pos); + return res; + } + //---------------------------------------------------------------------------- + std::string cut_off_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return str; + + res = str.substr(0, pos); + return res; + } + //---------------------------------------------------------------------------- +#ifdef _WIN32 + std::wstring utf8_to_utf16(const std::string& str) + { + if (str.empty()) + return {}; + int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); + if (wstr_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::wstring wstr(wstr_size, wchar_t{}); + if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return wstr; + } + std::string utf16_to_utf8(const std::wstring& wstr) + { + if (wstr.empty()) + return {}; + int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); + if (str_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::string str(str_size, char{}); + if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return str; + } +#endif } } diff --git a/contrib/epee/src/tiny_ini.cpp b/contrib/epee/src/tiny_ini.cpp new file mode 100644 index 000000000..577ebf7c6 --- /dev/null +++ b/contrib/epee/src/tiny_ini.cpp @@ -0,0 +1,46 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "string_tools.h" +#include <boost/regex.hpp> + +namespace epee +{ +namespace tiny_ini +{ + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) + { + std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; + const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) + return false; + res = result[2]; + string_tools::trim(res); + return true; + } +} +} diff --git a/contrib/epee/tests/src/CMakeLists.txt b/contrib/epee/tests/src/CMakeLists.txt index 4807fa7ea..0c42f87ca 100644 --- a/contrib/epee/tests/src/CMakeLists.txt +++ b/contrib/epee/tests/src/CMakeLists.txt @@ -1,5 +1,5 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) set(Boost_USE_MULTITHREADED ON) diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index e2e1d0b94..0a40d4608 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -36,6 +36,10 @@ This guide explains how to set up the environment, and how to start the builds. You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. +```bash +sudo adduser gitianuser +sudo usermod -aG sudo gitianuser +``` LXC --- @@ -83,7 +87,7 @@ Docker Prepare for building with docker: ```bash -sudo apt-get install git make curl docker.io +sudo bash -c 'apt-get update && apt-get upgrade -y && apt-get install git curl docker.io' ``` Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): @@ -96,13 +100,12 @@ sudo usermod -aG docker gitianuser Optionally add yourself to the docker group. Note that this will give docker root access to your system. ```bash -sudo usermod -aG docker gitianuser +sudo usermod -aG docker $USER ``` Manual Building ------------------- -The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. ======= The script automatically installs some packages with apt. If you are not running it on a debian-like system, pass `--no-apt` along with the other arguments to it. It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. @@ -122,17 +125,23 @@ cp monero/contrib/gitian/gitian-build.py . ### Setup the required environment -Setup for LXC: +Common setup part: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 +su - gitianuser -./gitian-build.py --setup $GH_USER $VERSION +GH_USER=YOUR_GITHUB_USER_NAME +VERSION=v0.17.2.0 ``` Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. +Setup for LXC: + +```bash +./gitian-build.py --setup $GH_USER $VERSION +``` + Setup for docker: ```bash @@ -145,8 +154,10 @@ fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) or pass the signed assert file back to your build machine. ```bash -git clone git@github.com:monero-project/gitian.sigs.git -git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git +git clone https://github.com/monero-project/gitian.sigs/ +pushd gitian.sigs +git remote add $GH_USER https://github.com/$GH_USER/gitian.sigs +popd ``` Build the binaries @@ -154,13 +165,26 @@ Build the binaries **Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. +Currently working MacOS solution: + +```bash +curl -O https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz +mv MacOSX10.11.sdk.tar.gz builder/inputs +``` + To build the most recent tag (pass in `--docker` if using docker): ```bash ./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION ``` -To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. +To speed up the build, use `-j 5 --memory 10000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 10000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. A good rule of thumb is, that Monero currently needs about 2 GB of RAM per core. + +A full example for `docker` would look like the following: + +```bash +./gitian-build.py -j 5 --memory 10000 --docker --detach-sign --no-commit --build $GH_USER $VERSION +``` If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. @@ -171,6 +195,22 @@ Take a look in the assert files and note the SHA256 checksums listed there. You should verify that the checksum that is listed matches each of the binaries you actually built. This may be done on Linux using the `sha256sum` command or on MacOS using `shasum --algorithm 256` for example. +An example script to verify the checksums would be: + +```bash +pushd out/${VERSION} + +for ASSERT in ../../sigs/${VERSION}-*/*/*.assert; do + if ! sha256sum --ignore-missing -c "${ASSERT}" ; then + echo "FAILED for ${ASSERT} ! Please inspect manually." + fi +done + +popd +``` + +Don't ignore the incorrect formatting of the found assert files. These files you'll have to compare manually (currently OSX and FreeBSD). + You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. @@ -181,14 +221,7 @@ Signing assert files If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 - -gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert -gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert -gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert -gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert -gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert +for ASSERT in sigs/${VERSION}-*/*/*.assert; do gpg --detach-sign ${ASSERT}; done ``` This will create a `.sig` file for each `.assert` file above (2 files for each platform). @@ -201,6 +234,7 @@ Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash +cd gitian.sigs git checkout -b $VERSION # add your assert and sig files... git commit -S -a -m "Add $GH_USER $VERSION" diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index a8140a8a6..6bf936958 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -94,10 +94,6 @@ def build(): os.chdir('builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) - subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) - subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', 'inputs/monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) rebuild() diff --git a/docs/COMPILING_DEBUGGING_TESTING.md b/docs/COMPILING_DEBUGGING_TESTING.md new file mode 100644 index 000000000..f5c202303 --- /dev/null +++ b/docs/COMPILING_DEBUGGING_TESTING.md @@ -0,0 +1,86 @@ +# Compiling, debugging and testing efficiently + +This document describes ways of compiling, debugging and testing efficiently for various use cases. +The intented audience are developers, who want to leverage newly added tricks to Monero via `CMake`. The document will lower the entry point for these developers. +Before reading this document, please consult section "Build instructions" in the main README.md. +Some information from README.md will be repeated here, but the aim is to go beyond it. + +## Basic compilation + +Monero can be compiled via the main `Makefile`, using one of several targets listed there. +The targets are actually presets for `CMake` calls with various options, plus `make` commands for building or in some cases `make test` for testing. +It is possible to extract these `CMake` calls and modify them for your specific needs. For example, a minimal external cmake command to compile Monero, executed from within a newly created build directory could look like: + +`cmake -S "$DIR_SRC" -DCMAKE_BUILD_TYPE=Release && make` + +where the variable `DIR_SRC` is expected to store the path to the Monero source code. + +## Use cases + +### Test Driven Development (TDD) - shared libraries for release builds + +Building shared libraries spares a lot of disk space and linkage time. By default only the debug builds produce shared libraries. If you'd like to produce dynamic libraries for the release build for the same reasons as it's being done for the debug version, then you need to add the `BUILD_SHARED_LIBS=ON` flag to the `CMake` call, like the following: + +`cmake -S "$DIR_SRC" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON && make` + +A perfect use case for the above call is following the Test Driven Development (TDD) principles. In a nutshell, you'd first write a couple of tests, which describe the (new) requirements of the class/method that you're about to write or modify. The tests will typically compile for quite a long time, so ideally write them once. After you're done with the tests, the only thing left to do is to keep modifying the implementation for as long as the tests are failing. If the implementation is contained properly within a .cpp file, then the only time cost to be paid will be compiling the single source file and generating the implementation's shared library. The test itself will not have to be touched and will pick up the new version of the implementation (via the shared library) upon the next execution of the test. + +### Project generation for IDEs + +CMake allows to generate project files for many IDEs. The list of supported project files can be obtained by writing in the console: + +`cmake -G` + +For instance, in order to generate Makefiles and project files for the Code::Blocks IDE, this part of the call would look like the following: + +`cmake -G "CodeBlocks - Unix Makefiles" (...)` + +The additional artifact of the above call is the `monero.cbp` Code::Blocks project file in the build directory. + +### Debugging in Code::Blocks (CB) + +First prepare the build directory for debugging using the following example command, assuming, that the path to the source dir is being held in the DIR_SRC variable, and using 2 cores: + +`cmake -S "$DIR_SRC" -G "CodeBlocks - Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON && make -j 2` + +After a successful build, open the `monero.cbp` with CB. From the CB's menu bar select the target, that you want debug. Assuming these are unit tests: + +`Build -> Select target -> Select target -> unit_tests` + +In order to lower the turnaround times, we will run a specific portion of code of interest, without having to go through all the time costly initialization and execution of unrelated parts. For this we'll use GTest's capabilities of test filtering. From the build directory run the following command to learn all the registered tests: + +`tests/unit_tests/unit_tests --gtest_list_tests` + +For example, if you're only interested in logging, you'd find in the list the label `logging.` and its subtests. To execute all the logging tests, you'd write in the console: + +`tests/unit_tests/unit_tests --gtest_filter="logging.*"` + +This parameter is what we need to transfer to CB, in order to reflect the same behaviour in the CB's debugger. From the main menu select: + +`Project -> Set program's arguments...` + +Then in the `Program's arguments` textbox you'd write in this case: + +`--gtest_filter="logging.*"` + +Verify if the expected UTs are being properly executed with `F9` or select: + +`Build -> Build and run` + +If everything looks fine, then after setting some breakpoints of your choice, the target is ready for debugging in CB via: + +`Debug -> Start/Continue` + +## To be done (and merged): +### Multihost parallel compilation +https://github.com/monero-project/monero/pull/7160 + +### Faster core_tests with caching +https://github.com/monero-project/monero/pull/5821 + +### Precompiled headers +https://github.com/monero-project/monero/pull/7216 + +### Unity builds +https://github.com/monero-project/monero/pull/7217 + diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 7b184c00a..e12295412 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -15,7 +15,7 @@ of software solid and usable. Patches are preferably to be sent via a Github pull request. If that can't be done, patches in "git format-patch" format can be sent (eg, posted to fpaste.org with a long enough timeout and a link -posted to #monero-dev on irc.freenode.net). +posted to #monero-dev on irc.libera.chat). Patches should be self contained. A good rule of thumb is to have one patch per separate issue, feature, or logical change. Also, no diff --git a/docs/README.i18n.md b/docs/README.i18n.md index 5df277624..0a3dc23a9 100644 --- a/docs/README.i18n.md +++ b/docs/README.i18n.md @@ -1,7 +1,7 @@ Monero daemon internationalization ================================== -The Monero command line tools can be translated in various languages. If you wish to contribute and need help/support, contact the [Monero Localization Workgroup on Taiga](https://taiga.getmonero.org/project/erciccione-monero-localization/) or come chat on `#monero-translations` (Freenode/IRC, riot/matrix, MatterMost) +The Monero command line tools can be translated in various languages. If you wish to contribute and need help/support, contact the [Monero Localization Workgroup on Taiga](https://taiga.getmonero.org/project/erciccione-monero-localization/) or come chat on `#monero-translations` (Libera/IRC, riot/matrix, MatterMost) In order to use the same translation workflow as the [Monero Core GUI](https://github.com/monero-project/monero-gui), they use Qt Linguist translation files. However, to avoid the dependencies on Qt this normally implies, they use a custom loader to read those files at runtime. diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 8699b445e..6314d5775 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -4882,9 +4882,6 @@ mdb_env_open2(MDB_env *env, int prev) #endif env->me_maxpg = env->me_mapsize / env->me_psize; - if (env->me_txns) - env->me_txns->mti_txnid = meta.mm_txnid; - #if MDB_DEBUG { MDB_meta *meta = mdb_env_pick_meta(env); @@ -4984,6 +4981,9 @@ static int ESECT mdb_env_share_locks(MDB_env *env, int *excl) { int rc = 0; + MDB_meta *meta = mdb_env_pick_meta(env); + + env->me_txns->mti_txnid = meta->mm_txnid; #ifdef _WIN32 { @@ -7867,7 +7867,7 @@ put_sub: xdata.mv_size = 0; xdata.mv_data = ""; leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (flags == MDB_CURRENT) { + if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) { xflags = MDB_CURRENT|MDB_NOSPILL; } else { mdb_xcursor_init1(mc, leaf); diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 35fb86552..9aa4c08bc 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -26,17 +26,22 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.5) project(easylogging CXX) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +monero_enable_coverage() find_package(Threads) find_package(Backtrace) +monero_find_all_headers(EASYLOGGING_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}") + add_library(easylogging - easylogging++.cc) + easylogging++.cc + ${EASYLOGGING_HEADERS} + ) include_directories("${CMAKE_CURRENT_SOURCE_DIR}") include_directories("${CMAKE_CURRENT_BINARY_DIR}") diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index bf877c018..267770074 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -17,6 +17,7 @@ #define EASYLOGGING_CC #include "easylogging++.h" +#include <atomic> #include <unistd.h> #if defined(AUTO_INITIALIZE_EASYLOGGINGPP) @@ -713,9 +714,8 @@ Logger::Logger(const std::string& id, const Configurations& configurations, } Logger::Logger(const Logger& logger) { - base::utils::safeDelete(m_typedConfigurations); m_id = logger.m_id; - m_typedConfigurations = logger.m_typedConfigurations; + m_typedConfigurations = logger.m_typedConfigurations ? new base::TypedConfigurations(*logger.m_typedConfigurations) : nullptr; m_parentApplicationName = logger.m_parentApplicationName; m_isConfigured = logger.m_isConfigured; m_configurations = logger.m_configurations; @@ -727,7 +727,7 @@ Logger& Logger::operator=(const Logger& logger) { if (&logger != this) { base::utils::safeDelete(m_typedConfigurations); m_id = logger.m_id; - m_typedConfigurations = logger.m_typedConfigurations; + m_typedConfigurations = logger.m_typedConfigurations ? new base::TypedConfigurations(*logger.m_typedConfigurations) : nullptr; m_parentApplicationName = logger.m_parentApplicationName; m_isConfigured = logger.m_isConfigured; m_configurations = logger.m_configurations; @@ -2035,7 +2035,7 @@ void RegisteredLoggers::unsafeFlushAll(void) { // VRegistry -VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags), m_lowest_priority(INT_MAX) { +VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags) { } /// @brief Sets verbose level. Accepted range is 0-9 @@ -2131,18 +2131,30 @@ static int priority(Level level) { return 7; } +namespace +{ + std::atomic<int> s_lowest_priority{INT_MAX}; +} + +void VRegistry::clearCategories(void) { + const base::threading::ScopedLock scopedLock(lock()); + m_categories.clear(); + m_cached_allowed_categories.clear(); + s_lowest_priority = INT_MAX; +} + void VRegistry::setCategories(const char* categories, bool clear) { base::threading::ScopedLock scopedLock(lock()); auto insert = [&](std::stringstream& ss, Level level) { m_categories.push_back(std::make_pair(ss.str(), level)); m_cached_allowed_categories.clear(); int pri = priority(level); - if (pri > m_lowest_priority) - m_lowest_priority = pri; + if (pri > s_lowest_priority) + s_lowest_priority = pri; }; if (clear) { - m_lowest_priority = 0; + s_lowest_priority = 0; m_categories.clear(); m_cached_allowed_categories.clear(); m_categoriesString.clear(); @@ -2200,9 +2212,9 @@ std::string VRegistry::getCategories() { } bool VRegistry::allowed(Level level, const std::string &category) { - const int pri = priority(level); - if (pri > m_lowest_priority) - return false; + return priority_allowed(priority(level), category); +} +bool VRegistry::priority_allowed(const int pri, const std::string &category) { base::threading::ScopedLock scopedLock(lock()); const std::map<std::string, int>::const_iterator it = m_cached_allowed_categories.find(category); if (it != m_cached_allowed_categories.end()) @@ -2477,20 +2489,20 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { template<typename Transform> -static inline std::string utf8canonical(const std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) +static inline void utf8canonical(std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) { - std::string sc = ""; size_t avail = s.size(); const char *ptr = s.data(); wint_t cp = 0; - int bytes = 1; + int rbytes = 1, bytes = -1; char wbuf[8], *wptr; + size_t w_offset = 0; while (avail--) { if ((*ptr & 0x80) == 0) { cp = *ptr++; - bytes = 1; + rbytes = 1; } else if ((*ptr & 0xe0) == 0xc0) { @@ -2499,7 +2511,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp = (*ptr++ & 0x1f) << 6; cp |= *ptr++ & 0x3f; --avail; - bytes = 2; + rbytes = 2; } else if ((*ptr & 0xf0) == 0xe0) { @@ -2509,7 +2521,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp |= (*ptr++ & 0x3f) << 6; cp |= *ptr++ & 0x3f; avail -= 2; - bytes = 3; + rbytes = 3; } else if ((*ptr & 0xf8) == 0xf0) { @@ -2520,7 +2532,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp |= (*ptr++ & 0x3f) << 6; cp |= *ptr++ & 0x3f; avail -= 3; - bytes = 4; + rbytes = 4; } else throw std::runtime_error("Invalid UTF-8"); @@ -2537,6 +2549,9 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w else throw std::runtime_error("Invalid code point UTF-8 transformation"); + if (bytes > rbytes) + throw std::runtime_error("In place sanitization requires replacements to not take more space than the original code points"); + wptr = wbuf; switch (bytes) { @@ -2547,16 +2562,17 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w default: throw std::runtime_error("Invalid UTF-8"); } *wptr = 0; - sc.append(wbuf, bytes); + memcpy(&s[w_offset], wbuf, bytes); + w_offset += bytes; cp = 0; bytes = 1; } - return sc; + s.resize(w_offset); } void sanitize(std::string &s) { - s = utf8canonical(s, [](wint_t c)->wint_t { + utf8canonical(s, [](wint_t c)->wint_t { if (c == 9 || c == 10 || c == 13) return c; if (c < 0x20) @@ -3335,6 +3351,14 @@ void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, c // Loggers +bool Loggers::allowed(Level level, const char* cat) +{ + const int pri = base::priority(level); + if (pri > base::s_lowest_priority) + return false; + return ELPP->vRegistry()->priority_allowed(pri, std::string{cat}); +} + Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); } diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 0b65461bc..c4a88339f 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -412,7 +412,6 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #include <sstream> #include <memory> #include <type_traits> -#include <atomic> #if ELPP_THREADING_ENABLED # if ELPP_USE_STD_THREADING # include <mutex> @@ -2464,12 +2463,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { return m_level; } - inline void clearCategories(void) { - base::threading::ScopedLock scopedLock(lock()); - m_categories.clear(); - m_cached_allowed_categories.clear(); - m_lowest_priority = INT_MAX; - } + void clearCategories(void); inline void clearModules(void) { base::threading::ScopedLock scopedLock(lock()); @@ -2482,6 +2476,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { void setModules(const char* modules); + bool priority_allowed(int priority, const std::string &category); bool allowed(Level level, const std::string &category); bool allowed(base::type::VerboseLevel vlevel, const char* file); @@ -2513,7 +2508,6 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { std::map<std::string, int> m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; - std::atomic<int> m_lowest_priority; }; } // namespace base class LogMessage { @@ -3868,6 +3862,8 @@ class Helpers : base::StaticClass { /// @brief Static helpers to deal with loggers and their configurations class Loggers : base::StaticClass { public: + /// @brief Determines whether logging will occur at this level and category + static bool allowed(Level leve, const char* cat); /// @brief Gets existing or registers new logger static Logger* getLogger(const std::string& identity, bool registerIfNotAvailable = true); /// @brief Changes default log builder for future loggers diff --git a/external/randomx b/external/randomx -Subproject 5ce5f4906c1eb166be980f6d83cc80f4112ffc2 +Subproject fe4324e8c0c035fec3affd6e4c49241c2e5b995 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9904c5de7..aaaae3a09 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -77,30 +77,7 @@ function (monero_add_executable name) enable_stack_trace("${name}") monero_set_target_no_relink("${name}") -endfunction () - -function (monero_add_library name) - monero_add_library_with_deps(NAME "${name}" SOURCES ${ARGN}) -endfunction() - -function (monero_add_library_with_deps) - cmake_parse_arguments(MONERO_ADD_LIBRARY "" "NAME" "DEPENDS;SOURCES" ${ARGN}) - source_group("${MONERO_ADD_LIBRARY_NAME}" FILES ${MONERO_ADD_LIBRARY_SOURCES}) - - # Define a ("virtual") object library and an actual library that links those - # objects together. The virtual libraries can be arbitrarily combined to link - # any subset of objects into one library archive. This is used for releasing - # libwallet, which combines multiple components. - set(objlib obj_${MONERO_ADD_LIBRARY_NAME}) - add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES}) - add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>) - monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}") - if (MONERO_ADD_LIBRARY_DEPENDS) - add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS}) - endif() - set_property(TARGET "${MONERO_ADD_LIBRARY_NAME}" PROPERTY FOLDER "libs") - target_compile_definitions(${objlib} - PRIVATE $<TARGET_PROPERTY:${MONERO_ADD_LIBRARY_NAME},INTERFACE_COMPILE_DEFINITIONS>) + monero_set_target_strip ("${name}") endfunction () include(Version) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 5e12fa8ec..a84a4148d 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -216,15 +216,8 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair } else { - LOG_PRINT_L1("Unsupported input type, removing key images and aborting transaction addition"); - for (const txin_v& tx_input : tx.vin) - { - if (tx_input.type() == typeid(txin_to_key)) - { - remove_spent_key(boost::get<txin_to_key>(tx_input).k_image); - } - } - return; + LOG_PRINT_L1("Unsupported input type, aborting transaction addition"); + throw std::runtime_error("Unexpected input type, aborting"); } } diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 5f3b495b0..1d7b10648 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1354,7 +1354,8 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) m_folder = filename; - check_mmap_support(); + try { check_mmap_support(); } + catch(...) { MERROR("Failed to check for mmap support, proceeding"); } #ifdef __OpenBSD__ if ((mdb_flags & MDB_WRITEMAP) == 0) { diff --git a/src/blockchain_utilities/blockchain-stats-readme.md b/src/blockchain_utilities/blockchain-stats-readme.md new file mode 100644 index 000000000..e02cd5560 --- /dev/null +++ b/src/blockchain_utilities/blockchain-stats-readme.md @@ -0,0 +1,54 @@ +# Monero Blockchain Stats + +Monero Blockchain Stats utlity exports daily statistics for the monero blockchain from creation through current state. + +## Usage: + +See also the utility's help option. `monero-blockchain-stats --help` + +From the command line run: + +`$ monero-blockchain-stats` + +This loads the existing blockchain and prints the results to the terminal. Default printed data includes Blocks per Day, Total Blocks, Transactions per Day, Total Transactions, Bytes per Day and Total Bytes. The format of the output is in tab delimited csv which is printed to the console. Piping the output of the command to a csv file allows for saving the output of the utilty to a file. +i.e. `monero-blockchain-stats > stats.csv` + +### Options +`--data-dir arg` +to specify location of blockchain storage + +`--testnet` +Run on testnet. + +`--stagenet` +Run on stagenet. + +`--log-level arg` +0-4 or categories + +`--block-start arg (=0)` +start at block number + +`--block-stop arg (=0)` +Stop at block number + +`--with-inputs` +with input stats + +`--with-outputs` +with output stats + +`--with-ringsize` +with ringsize stats + +`--with-hours` +with txns per hour + +`--with-emission` +with coin emission + +`--with-fees` +with txn fees + +`--with-diff` +with difficulty diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index b1b238427..99a84606d 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> +#include <boost/filesystem/path.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index e439895bf..d53251fd3 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" @@ -387,9 +388,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id cryptonote::transaction_prefix tx; blobdata bd; bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); - std::stringstream ss; - ss << bd; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(bd)}; bool r = do_serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 6199586bf..8c3c3a009 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem/path.hpp> #include <boost/algorithm/string.hpp> #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index df2662444..8d81ef54d 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -146,7 +146,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block if (!parse_and_validate_block_from_blob(b.block, block)) { MERROR("Failed to parse block: " - << epee::string_tools::pod_to_hex(get_blob_hash(b.block))); + << epee::string_tools::buff_to_hex_nodelimer(b.block)); core.cleanup_handle_incoming_blocks(); return 1; } @@ -177,8 +177,11 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block core.handle_incoming_tx(tx_blob, tvc, relay_method::block, true); if(tvc.m_verifivation_failed) { - MERROR("transaction verification failed, tx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob.blob))); + cryptonote::transaction transaction; + if (cryptonote::parse_and_validate_tx_from_blob(tx_blob.blob, transaction)) + MERROR("Transaction verification failed, tx_id = " << cryptonote::get_transaction_hash(transaction)); + else + MERROR("Transaction verification failed, transaction is unparsable"); core.cleanup_handle_incoming_blocks(); return 1; } @@ -192,8 +195,11 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block if(bvc.m_verifivation_failed) { - MERROR("Block verification failed, id = " - << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block))); + cryptonote::block block; + if (cryptonote::parse_and_validate_block_from_blob(block_entry.block, block)) + MERROR("Block verification failed, id = " << cryptonote::get_block_hash(block)); + else + MERROR("Block verification failed, block is unparsable"); core.cleanup_handle_incoming_blocks(); return 1; } diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 9a9d58c46..b1c599f3a 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -29,6 +29,8 @@ #include <array> #include <lmdb.h> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/pruning.h" #include "cryptonote_core/cryptonote_core.h" diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index 1a54778a7..78a662134 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "serialization/crypto.h" #include "cryptonote_core/tx_pool.h" diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 1f728b4e5..5f5ca6abf 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_basic/cryptonote_boost_serialization.h" diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 095e6c503..8356ef420 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem/path.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_core/tx_pool.h" diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index 445596a66..43af5023d 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -28,20 +28,32 @@ set(GENERATED_SOURCES "") +set(GENERATOR "${CMAKE_CURRENT_BINARY_DIR}/blocks_generator.cmake") +file(GENERATE OUTPUT ${GENERATOR} CONTENT [=[ +file(READ "${INPUT_DAT_FILE}" DATA HEX) +string(REGEX REPLACE "[0-9a-fA-F][0-9a-fA-F]" "0x\\0," DATA "${DATA}") +file(WRITE "${OUTPUT_C_SOURCE}" " +#include <stddef.h> +const unsigned char ${BLOB_NAME}[]={ + ${DATA} +}; +const size_t ${BLOB_NAME}_len = sizeof(${BLOB_NAME}); +" +) +]=]) foreach(BLOB_NAME checkpoints testnet_blocks stagenet_blocks) set(OUTPUT_C_SOURCE "generated_${BLOB_NAME}.c") list(APPEND GENERATED_SOURCES ${OUTPUT_C_SOURCE}) set(INPUT_DAT_FILE "${BLOB_NAME}.dat") add_custom_command( - OUTPUT ${OUTPUT_C_SOURCE} - MAIN_DEPENDENCY ${INPUT_DAT_FILE} - COMMAND - cd ${CMAKE_CURRENT_BINARY_DIR} && - echo "'#include\t<stddef.h>'" > ${OUTPUT_C_SOURCE} && - echo "'const\tunsigned\tchar\t${BLOB_NAME}[]={'" >> ${OUTPUT_C_SOURCE} && - od -v -An -tx1 ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE} | sed -e "'s/[0-9a-fA-F]\\{1,\\}/0x&,/g'" -e "'$$s/.$$//'" >> ${OUTPUT_C_SOURCE} && - echo "'};'" >> ${OUTPUT_C_SOURCE} && - echo "'const\tsize_t\t${BLOB_NAME}_len\t=\tsizeof(${BLOB_NAME});'" >> ${OUTPUT_C_SOURCE} + OUTPUT ${OUTPUT_C_SOURCE} + MAIN_DEPENDENCY ${INPUT_DAT_FILE} + DEPENDS ${GENERATOR} + COMMAND ${CMAKE_COMMAND} + "-DINPUT_DAT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE}" + "-DBLOB_NAME=${BLOB_NAME}" + "-DOUTPUT_C_SOURCE=${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_C_SOURCE}" + "-P${GENERATOR}" ) endforeach() diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index c88a630cc..6b48d8723 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -34,6 +34,8 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <functional> #include <vector> diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index c31d1dd96..f0b617798 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -37,6 +37,7 @@ #include <boost/thread/mutex.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/optional.hpp> +#include <boost/utility/string_ref.hpp> using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY @@ -124,6 +125,7 @@ static const char *get_record_name(int record_type) case DNS_TYPE_A: return "A"; case DNS_TYPE_TXT: return "TXT"; case DNS_TYPE_AAAA: return "AAAA"; + case DNS_TYPE_TLSA: return "TLSA"; default: return "unknown"; } } @@ -186,6 +188,13 @@ boost::optional<std::string> txt_to_string(const char* src, size_t len) return std::string(src+1, len-1); } +boost::optional<std::string> tlsa_to_string(const char* src, size_t len) +{ + if (len < 4) + return boost::none; + return std::string(src, len); +} + // custom smart pointer. // TODO: see if std::auto_ptr and the like support custom destructors template<typename type, void (*freefunc)(type*)> @@ -326,11 +335,15 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec // destructor takes care of cleanup ub_result_ptr result; + MDEBUG("Performing DNSSEC " << get_record_name(record_type) << " record query for " << url); + // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, string_copy(url.c_str()), record_type, DNS_CLASS_IN, &result)) { dnssec_available = (result->secure || result->bogus); dnssec_valid = result->secure && !result->bogus; + if (dnssec_available && !dnssec_valid) + MWARNING("Invalid DNSSEC " << get_record_name(record_type) << " record signature for " << url << ": " << result->why_bogus); if (result->havedata) { for (size_t i=0; result->data[i] != NULL; i++) @@ -338,8 +351,9 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec boost::optional<std::string> res = (*reader)(result->data[i], result->len[i]); if (res) { - MINFO("Found \"" << *res << "\" in " << get_record_name(record_type) << " record for " << url); - addresses.push_back(*res); + // do not dump dns record directly from dns into log + MINFO("Found " << get_record_name(record_type) << " record for " << url); + addresses.push_back(std::move(*res)); } } } @@ -363,6 +377,17 @@ std::vector<std::string> DNSResolver::get_txt_record(const std::string& url, boo return get_record(url, DNS_TYPE_TXT, txt_to_string, dnssec_available, dnssec_valid); } +std::vector<std::string> DNSResolver::get_tlsa_tcp_record(const boost::string_ref url, const boost::string_ref port, bool& dnssec_available, bool& dnssec_valid) +{ + std::string service_addr; + service_addr.reserve(url.size() + port.size() + 7); + service_addr.push_back('_'); + service_addr.append(port.data(), port.size()); + service_addr.append("._tcp."); + service_addr.append(url.data(), url.size()); + return get_record(service_addr, DNS_TYPE_TLSA, tlsa_to_string, dnssec_available, dnssec_valid); +} + std::string DNSResolver::get_dns_format_from_oa_address(const std::string& oa_addr) { std::string addr(oa_addr); diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index 30c4cced2..99e91bc54 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -31,15 +31,17 @@ #include <string> #include <functional> #include <boost/optional/optional_fwd.hpp> +#include <boost/utility/string_ref_fwd.hpp> namespace tools { // RFC defines for record types and classes for DNS, gleaned from ldns source -const static int DNS_CLASS_IN = 1; -const static int DNS_TYPE_A = 1; -const static int DNS_TYPE_TXT = 16; -const static int DNS_TYPE_AAAA = 8; +constexpr const int DNS_CLASS_IN = 1; +constexpr const int DNS_TYPE_A = 1; +constexpr const int DNS_TYPE_TXT = 16; +constexpr const int DNS_TYPE_AAAA = 8; +constexpr const int DNS_TYPE_TLSA = 52; struct DNSResolverData; @@ -106,6 +108,17 @@ public: std::vector<std::string> get_txt_record(const std::string& url, bool& dnssec_available, bool& dnssec_valid); /** + * @brief gets all TLSA TCP records from a DNS query for the supplied URL; + * if no TLSA record present returns an empty vector. + * + * @param url A string containing a URL to query for + * @param port The service port number (as string) to query + * + * @return A vector of strings containing all TLSA records; or an empty vector + */ + std::vector<std::string> get_tlsa_tcp_record(boost::string_ref url, boost::string_ref port, bool& dnssec_available, bool& dnssec_valid); + + /** * @brief Gets a DNS address from OpenAlias format * * If the address looks good, but contains one @ symbol, replace that with a . diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index ebc367b3a..051220ee1 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -35,6 +35,10 @@ #include "common/i18n.h" #include "translation_files.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> +#include <algorithm> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 3b33fe90a..3f0f7d34b 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -116,6 +116,7 @@ endif() # cheat because cmake and ccache hate each other set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C) +set_property(SOURCE CryptonightR_template.S PROPERTY XCODE_EXPLICIT_FILE_TYPE sourcecode.asm) # Must be done last, because it references libraries in this directory add_subdirectory(wallet) diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index 7dfc5151d..1cd502994 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -87,6 +87,10 @@ void hash_extra_jh(const void *data, size_t length, char *hash); void hash_extra_skein(const void *data, size_t length, char *hash); void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash); +bool tree_path(size_t count, size_t idx, uint32_t *path); +bool tree_branch(const char (*hashes)[HASH_SIZE], size_t count, const char *hash, char (*branch)[HASH_SIZE], size_t *depth, uint32_t *path); +bool tree_branch_hash(const char hash[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path, char root[HASH_SIZE]); +bool is_branch_in_tree(const char hash[HASH_SIZE], const char root[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path); #define RX_BLOCK_VERSION 12 void rx_slow_hash_allocate_state(void); diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index fa35a32e2..801987e37 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -264,12 +264,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch cache = rx_sp->rs_cache; if (cache == NULL) { - if (cache == NULL) { + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); if (cache == NULL) { mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache"); - cache = randomx_alloc_cache(flags); } + } + if (cache == NULL) { + cache = randomx_alloc_cache(flags); if (cache == NULL) local_abort("Couldn't allocate RandomX cache"); } @@ -291,11 +293,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch CTHR_MUTEX_LOCK(rx_dataset_mutex); if (!rx_dataset_nomem) { if (rx_dataset == NULL) { - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); - if (rx_dataset == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); + if (rx_dataset == NULL) { + mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); + } } + if (rx_dataset == NULL) + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); if (rx_dataset != NULL) rx_initdata(rx_sp->rs_cache, miners, seedheight); } @@ -311,11 +316,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch } CTHR_MUTEX_UNLOCK(rx_dataset_mutex); } - rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) { //large pages failed - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { + rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); + if(rx_vm == NULL) { //large pages failed + mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); + } } + if (rx_vm == NULL) + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); if(rx_vm == NULL) {//fallback if everything fails flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 643e95121..8f3ea3339 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -104,3 +104,154 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { free(ints); } } + +bool tree_path(size_t count, size_t idx, uint32_t *path) +{ + if (count == 0) + return false; + + if (count == 1) { + *path = 0; + } else if (count == 2) { + *path = idx == 0 ? 0 : 1; + } else { + size_t i, j; + + *path = 0; + size_t cnt = tree_hash_cnt( count ); + + for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { + if (idx == i || idx == i+1) + { + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + } + assert(i == count); + + while (cnt > 2) { + cnt >>= 1; + for (i = 0, j = 0; j < cnt; i += 2, ++j) { + if (idx == i || idx == i + 1) + { + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + } + } + + if (idx == 0 || idx == 1) + { + *path = (*path << 1) | (idx == 0 ? 0 : 1); + idx = 0; + } + } + return true; +} + +bool tree_branch(const char (*hashes)[HASH_SIZE], size_t count, const char *hash, char (*branch)[HASH_SIZE], size_t *depth, uint32_t *path) +{ + size_t idx; + + if (count == 0) + return false; + + for (idx = 0; idx < count; ++idx) + if (!memcmp(hash, hashes[idx], HASH_SIZE)) + break; + if (idx == count) + return false; + + assert(count > 0); + if (count == 1) { + *depth = 0; + *path = 0; + } else if (count == 2) { + *depth = 1; + *path = idx == 0 ? 0 : 1; + memcpy(branch[0], hashes[idx ^ 1], HASH_SIZE); + } else { + size_t i, j; + + *depth = 0; + *path = 0; + size_t cnt = tree_hash_cnt( count ); + + char *ints = calloc(cnt, HASH_SIZE); // zero out as extra protection for using uninitialized mem + assert(ints); + + memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); + + for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { + if (idx == i || idx == i+1) + { + memcpy(branch[*depth], hashes[idx == i ? i + 1 : i], HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + cn_fast_hash(hashes[i], 64, ints + j * HASH_SIZE); + } + assert(i == count); + + while (cnt > 2) { + cnt >>= 1; + for (i = 0, j = 0; j < cnt; i += 2, ++j) { + if (idx == i || idx == i + 1) + { + memcpy(branch[*depth], ints + (idx == i ? i + 1 : i) * HASH_SIZE, HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + cn_fast_hash(ints + i * HASH_SIZE, 64, ints + j * HASH_SIZE); + } + } + + if (idx == 0 || idx == 1) + { + memcpy(branch[*depth], ints + (idx == 0 ? 1 : 0) * HASH_SIZE, HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == 0 ? 0 : 1); + idx = 0; + } + + free(ints); + } + return true; +} + +bool tree_branch_hash(const char hash[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path, char root[HASH_SIZE]) +{ + size_t d; + char partial[HASH_SIZE]; + + memcpy(partial, hash, HASH_SIZE); + + for (d = 0; d < depth; ++d) + { + char buffer[2 * HASH_SIZE]; + if ((path >> (depth - d - 1)) & 1) + { + memcpy(buffer, branch[d], HASH_SIZE); + memcpy(buffer + HASH_SIZE, partial, HASH_SIZE); + } + else + { + memcpy(buffer, partial, HASH_SIZE); + memcpy(buffer + HASH_SIZE, branch[d], HASH_SIZE); + } + cn_fast_hash(buffer, 2 * HASH_SIZE, partial); + } + + memcpy(root, partial, HASH_SIZE); + return true; +} + +bool is_branch_in_tree(const char hash[HASH_SIZE], const char root[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path) +{ + char res[HASH_SIZE]; + if (!tree_branch_hash(hash, branch, depth, path, res)) + return false; + return memcmp(res, root, HASH_SIZE) == 0; +} diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index 5286256c7..e386ec4ea 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -36,6 +36,14 @@ if(APPLE) endif() endif() +monero_add_library(cryptonote_format_utils_basic + cryptonote_format_utils_basic.cpp +) +target_link_libraries(cryptonote_format_utils_basic + PUBLIC + cncrypto +) + set(cryptonote_basic_sources account.cpp connection_context.cpp @@ -43,6 +51,7 @@ set(cryptonote_basic_sources cryptonote_format_utils.cpp difficulty.cpp hardfork.cpp + merge_mining.cpp miner.cpp) set(cryptonote_basic_headers) @@ -57,6 +66,7 @@ set(cryptonote_basic_private_headers cryptonote_format_utils.h difficulty.h hardfork.h + merge_mining.h miner.h tx_extra.h verification_context.h) @@ -72,6 +82,7 @@ target_link_libraries(cryptonote_basic common cncrypto checkpoints + cryptonote_format_utils_basic device ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index a7d688300..ee26a0c07 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -77,6 +77,8 @@ namespace cryptonote int m_expect_response; uint64_t m_expect_height; size_t m_num_requested; + epee::copyable_atomic m_new_stripe_notification{0}; + epee::copyable_atomic m_idle_peer_notification{0}; }; inline std::string get_protocol_state_string(cryptonote_connection_context::state s) diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index c70ae1df1..6394a7071 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -152,10 +152,6 @@ namespace cryptonote }; - template<typename T> static inline unsigned int getpos(T &ar) { return 0; } - template<> inline unsigned int getpos(binary_archive<true> &ar) { return ar.stream().tellp(); } - template<> inline unsigned int getpos(binary_archive<false> &ar) { return ar.stream().tellg(); } - class transaction_prefix { @@ -236,17 +232,17 @@ namespace cryptonote set_blob_size_valid(false); } - const unsigned int start_pos = getpos(ar); + const auto start_pos = ar.getpos(); FIELDS(*static_cast<transaction_prefix *>(this)) if (std::is_same<Archive<W>, binary_archive<W>>()) - prefix_size = getpos(ar) - start_pos; + prefix_size = ar.getpos() - start_pos; if (version == 1) { if (std::is_same<Archive<W>, binary_archive<W>>()) - unprunable_size = getpos(ar) - start_pos; + unprunable_size = ar.getpos() - start_pos; ar.tag("signatures"); ar.begin_array(); @@ -284,11 +280,11 @@ namespace cryptonote { ar.begin_object(); bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); - if (!r || !ar.stream().good()) return false; + if (!r || !ar.good()) return false; ar.end_object(); if (std::is_same<Archive<W>, binary_archive<W>>()) - unprunable_size = getpos(ar) - start_pos; + unprunable_size = ar.getpos() - start_pos; if (!pruned && rct_signatures.type != rct::RCTTypeNull) { @@ -296,7 +292,7 @@ namespace cryptonote ar.begin_object(); r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin.size(), vout.size(), vin.size() > 0 && vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(vin[0]).key_offsets.size() - 1 : 0); - if (!r || !ar.stream().good()) return false; + if (!r || !ar.good()) return false; ar.end_object(); } } @@ -320,13 +316,13 @@ namespace cryptonote { ar.begin_object(); bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); - if (!r || !ar.stream().good()) return false; + if (!r || !ar.good()) return false; ar.end_object(); } } if (!typename Archive<W>::is_saving()) pruned = true; - return ar.stream().good(); + return ar.good(); } private: diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 1ef6590eb..17adcdc35 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include "wipeable_string.h" #include "string_tools.h" +#include "string_tools_lexical.h" #include "serialization/string.h" #include "cryptonote_format_utils.h" #include "cryptonote_config.h" @@ -138,22 +139,6 @@ namespace cryptonote return h; } - //--------------------------------------------------------------- - void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) - { - std::ostringstream s; - binary_archive<true> a(s); - ::serialization::serialize(a, const_cast<transaction_prefix&>(tx)); - crypto::cn_fast_hash(s.str().data(), s.str().size(), h); - } - //--------------------------------------------------------------- - crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx) - { - crypto::hash h = null_hash; - get_transaction_prefix_hash(tx, h); - return h; - } - //--------------------------------------------------------------- bool expand_transaction_1(transaction &tx, bool base_only) { if (tx.version >= 2 && !is_coinbase(tx)) @@ -210,9 +195,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = ::serialization::serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data"); @@ -223,9 +206,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_base_from_blob(const blobdata_ref& tx_blob, transaction& tx) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = tx.serialize_base(ba); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data"); @@ -235,9 +216,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_prefix_from_blob(const blobdata_ref& tx_blob, transaction_prefix& tx) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = ::serialization::serialize_noeof(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction prefix from blob"); return true; @@ -245,9 +224,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = ::serialization::serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data"); @@ -531,22 +508,15 @@ namespace cryptonote if(tx_extra.empty()) return true; - std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); - std::istringstream iss(extra_str); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::to_span(tx_extra)}; - bool eof = false; - while (!eof) + do { tx_extra_field field; bool r = ::do_serialize(ar, field); CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); tx_extra_fields.push_back(field); - - std::ios_base::iostate state = iss.rdstate(); - eof = (EOF == iss.peek()); - iss.clear(state); - } + } while (!ar.eof()); CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); return true; @@ -577,13 +547,10 @@ namespace cryptonote return true; } - std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); - std::istringstream iss(extra_str); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::to_span(tx_extra)}; - bool eof = false; size_t processed = 0; - while (!eof) + do { tx_extra_field field; bool r = ::do_serialize(ar, field); @@ -595,12 +562,8 @@ namespace cryptonote break; } tx_extra_fields.push_back(field); - processed = iss.tellg(); - - std::ios_base::iostate state = iss.rdstate(); - eof = (EOF == iss.peek()); - iss.clear(state); - } + processed = ar.getpos(); + } while (!ar.eof()); if (!::serialization::check_stream_state(ar)) { MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); @@ -727,29 +690,42 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth) + { + CHECK_AND_ASSERT_MES(mm_merkle_tree_depth < 32, false, "merge mining merkle tree depth should be less than 32"); + size_t start_pos = tx_extra.size(); + tx_extra.resize(tx_extra.size() + 3 + 32); + //write tag + tx_extra[start_pos] = TX_EXTRA_MERGE_MINING_TAG; + //write data size + ++start_pos; + tx_extra[start_pos] = 33; + //write depth varint (always one byte here) + ++start_pos; + tx_extra[start_pos] = mm_merkle_tree_depth; + //write data + ++start_pos; + memcpy(&tx_extra[start_pos], &mm_merkle_root, 32); + return true; + } + //--------------------------------------------------------------- bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type) { if (tx_extra.empty()) return true; std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); - std::istringstream iss(extra_str); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(extra_str)}; std::ostringstream oss; binary_archive<true> newar(oss); - bool eof = false; - while (!eof) + do { tx_extra_field field; bool r = ::do_serialize(ar, field); CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); if (field.type() != type) ::do_serialize(newar, field); - - std::ios_base::iostate state = iss.rdstate(); - eof = (EOF == iss.peek()); - iss.clear(state); - } + } while (!ar.eof()); CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); tx_extra.clear(); std::string s = oss.str(); @@ -1337,9 +1313,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b, crypto::hash *block_hash) { - std::stringstream ss; - ss << b_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(b_blob)}; bool r = ::serialization::serialize(ba, b); CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob"); b.invalidate_hashes(); diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 636a88b9a..3fe4c44e7 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -83,6 +83,7 @@ namespace cryptonote std::vector<crypto::public_key> get_additional_tx_pub_keys_from_extra(const transaction_prefix& tx); bool add_additional_tx_pub_keys_to_extra(std::vector<uint8_t>& tx_extra, const std::vector<crypto::public_key>& additional_pub_keys); bool add_extra_nonce_to_tx_extra(std::vector<uint8_t>& tx_extra, const blobdata& extra_nonce); + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth); bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type); void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id); void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id); @@ -147,9 +148,7 @@ namespace cryptonote template<class t_object> bool t_serializable_object_from_blob(t_object& to, const blobdata& b_blob) { - std::stringstream ss; - ss << b_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(b_blob)}; bool r = ::serialization::serialize(ba, to); return r; } diff --git a/src/cryptonote_basic/cryptonote_format_utils_basic.cpp b/src/cryptonote_basic/cryptonote_format_utils_basic.cpp new file mode 100644 index 000000000..29130ce46 --- /dev/null +++ b/src/cryptonote_basic/cryptonote_format_utils_basic.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2014-2021, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "cryptonote_format_utils.h" + +namespace cryptonote +{ + void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction_prefix&>(tx)); + crypto::cn_fast_hash(s.str().data(), s.str().size(), h); + } + + crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx) + { + crypto::hash h = crypto::null_hash; + get_transaction_prefix_hash(tx, h); + return h; + } +} diff --git a/src/cryptonote_basic/merge_mining.cpp b/src/cryptonote_basic/merge_mining.cpp new file mode 100644 index 000000000..fcc74859f --- /dev/null +++ b/src/cryptonote_basic/merge_mining.cpp @@ -0,0 +1,95 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string.h> +#include "misc_log_ex.h" +#include "int-util.h" +#include "crypto/crypto.h" +#include "common/util.h" +#include "merge_mining.h" + +using namespace epee; + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "cn.mm" + +using namespace crypto; + +namespace cryptonote +{ + +//--------------------------------------------------------------- +uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + + uint8_t buf[HASH_SIZE + sizeof(uint32_t) + 1]; + memcpy(buf, &id, HASH_SIZE); + uint32_t v = SWAP32LE(nonce); + memcpy(buf + HASH_SIZE, &v, sizeof(uint32_t)); + buf[HASH_SIZE + sizeof(uint32_t)] = config::HASH_KEY_MM_SLOT; + + crypto::hash res; + tools::sha256sum(buf, sizeof(buf), res); + v = *((const uint32_t*)&res); + return SWAP32LE(v) % n_aux_chains; +} +//--------------------------------------------------------------- +uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + CHECK_AND_ASSERT_THROW_MES(slot < n_aux_chains, "slot >= n_aux_chains"); + + uint32_t path = 0; + CHECK_AND_ASSERT_THROW_MES(tree_path(n_aux_chains, slot, &path), "Failed to get path from aux slot"); + return path; +} +//--------------------------------------------------------------- +uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + + // how many bits to we need to representing n_aux_chains - 1 + uint32_t n_bits = 1; + while ((1u << n_bits) < n_aux_chains && n_bits < 16) + ++n_bits; + CHECK_AND_ASSERT_THROW_MES(n_bits <= 16, "Way too many bits required"); + + const uint32_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (nonce << (3 + n_bits)); + return depth; +} +//--------------------------------------------------------------- +bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce) +{ + const uint32_t n_bits = 1 + (depth & 7); + n_aux_chains = 1 + (depth >> 3 & ((1 << n_bits) - 1)); + nonce = depth >> (3 + n_bits); + return true; +} +//--------------------------------------------------------------- +} diff --git a/src/cryptonote_basic/merge_mining.h b/src/cryptonote_basic/merge_mining.h new file mode 100644 index 000000000..378438f7c --- /dev/null +++ b/src/cryptonote_basic/merge_mining.h @@ -0,0 +1,40 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stdint.h> +#include "crypto/crypto.h" + +namespace cryptonote +{ + uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains); + uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains); + uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce); + bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce); +} diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 29f6dce5a..ae514aac6 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -44,6 +44,7 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" #include "boost/logic/tribool.hpp" +#include <boost/filesystem.hpp> #ifdef __APPLE__ #include <sys/times.h> diff --git a/src/cryptonote_basic/tx_extra.h b/src/cryptonote_basic/tx_extra.h index 50f2e1438..76efc22d3 100644 --- a/src/cryptonote_basic/tx_extra.h +++ b/src/cryptonote_basic/tx_extra.h @@ -57,11 +57,7 @@ namespace cryptonote // size - 1 - because of variant tag for (size = 1; size <= TX_EXTRA_PADDING_MAX_COUNT; ++size) { - std::ios_base::iostate state = ar.stream().rdstate(); - bool eof = EOF == ar.stream().peek(); - ar.stream().clear(state); - - if (eof) + if (ar.eof()) break; uint8_t zero; @@ -139,8 +135,7 @@ namespace cryptonote if(!::do_serialize(ar, field)) return false; - std::istringstream iss(field); - binary_archive<false> iar(iss); + binary_archive<false> iar{epee::strspan<std::uint8_t>(field)}; serialize_helper helper(*this); return ::serialization::serialize(iar, helper); } diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 6327f7379..915835d1b 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -234,6 +234,7 @@ namespace config const unsigned char HASH_KEY_CLSAG_AGG_0[] = "CLSAG_agg_0"; const unsigned char HASH_KEY_CLSAG_AGG_1[] = "CLSAG_agg_1"; const char HASH_KEY_MESSAGE_SIGNING[] = "MoneroMessageSignature"; + const unsigned char HASH_KEY_MM_SLOT = 'm'; namespace testnet { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index cdad39a2c..f0e6794b9 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -57,6 +57,7 @@ #include "common/notify.h" #include "common/varint.h" #include "common/pruning.h" +#include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -565,6 +566,8 @@ void Blockchain::pop_blocks(uint64_t nblocks) return; } + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); + if (stop_batch) m_db->batch_stop(); } @@ -642,7 +645,6 @@ block Blockchain::pop_block_from_blockchain() m_scan_table.clear(); m_blocks_txs_check.clear(); - CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); uint64_t top_block_height; crypto::hash top_block_hash = get_tail_id(top_block_height); m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); @@ -965,7 +967,7 @@ start: MGINFO("START DUMP"); MGINFO(ss.str()); MGINFO("END DUMP"); - MGINFO("Please send moneromooo on Freenode the contents of this log, from a couple dozen lines before START DUMP to END DUMP"); + MGINFO("Please send moneromooo on Libera.Chat the contents of this log, from a couple dozen lines before START DUMP to END DUMP"); } return diff; } @@ -1109,6 +1111,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, { pop_block_from_blockchain(); } + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); // make sure the hard fork object updates its current version m_hardfork->reorganize_from_chain_height(rollback_height); @@ -1159,6 +1162,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> block b = pop_block_from_blockchain(); disconnected_chain.push_front(b); } + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); auto split_height = m_db->height(); @@ -2608,7 +2612,7 @@ bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_ } //------------------------------------------------------------------ template<class t_ids_container, class t_tx_container, class t_missed_container> -bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2619,10 +2623,12 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container try { cryptonote::blobdata tx; - if (m_db->get_tx_blob(tx_hash, tx)) + bool res = pruned ? m_db->get_pruned_tx_blob(tx_hash, tx) : m_db->get_tx_blob(tx_hash, tx); + if (res) { txs.push_back(transaction()); - if (!parse_and_validate_tx_from_blob(tx, txs.back())) + res = pruned ? parse_and_validate_tx_base_from_blob(tx, txs.back()) : parse_and_validate_tx_from_blob(tx, txs.back()); + if (!res) { LOG_ERROR("Invalid transaction"); return false; @@ -5149,7 +5155,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete if (m_cancel) return false; - for (const auto &tx_blob : entry.txs) + for (size_t i = 0; i < entry.txs.size(); ++i) { if (tx_index >= txes.size()) SCAN_TABLE_QUIT("tx_index is out of sync"); @@ -5526,6 +5532,6 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ } namespace cryptonote { -template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; +template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 5291f1338..a0e7967de 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -721,7 +721,7 @@ namespace cryptonote template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; template<class t_ids_container, class t_tx_container, class t_missed_container> - bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; //debug functions diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 57104fd59..1da14221a 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -57,6 +57,8 @@ using namespace epee; #include "hardforks/hardforks.h" #include "version.h" +#include <boost/filesystem.hpp> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" @@ -427,9 +429,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const { - return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); + return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs, pruned); } //----------------------------------------------------------------------------------------------- bool core::get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const @@ -443,9 +445,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const { - return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); + return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs, pruned); } //----------------------------------------------------------------------------------------------- bool core::get_alternative_blocks(std::vector<block>& blocks) const @@ -1241,7 +1243,7 @@ namespace cryptonote std::vector<transaction> txs; std::vector<crypto::hash> missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); - this->get_transactions(b.tx_hashes, txs, missed_txs); + this->get_transactions(b.tx_hashes, txs, missed_txs, true); uint64_t tx_fee_amount = 0; for(const auto& tx: txs) { diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8891540a9..8478049f9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -385,7 +385,7 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const; /** * @copydoc Blockchain::get_transactions @@ -399,7 +399,7 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const; /** * @copydoc Blockchain::get_block_by_hash diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 7400c4328..f41c63a4b 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -622,8 +622,10 @@ namespace cryptonote if (need_additional_txkeys) { additional_tx_keys.clear(); - for (const auto &d: destinations) + for (size_t i = 0; i < destinations.size(); ++i) + { additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); + } } bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout); diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index dbdf409b5..73cdd31cd 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -44,10 +44,10 @@ namespace cryptonote typedef std::pair<uint64_t, rct::ctkey> output_entry; std::vector<output_entry> outputs; //index + key + optional ringct commitment - size_t real_output; //index in outputs vector of real output_entry + uint64_t real_output; //index in outputs vector of real output_entry crypto::public_key real_out_tx_key; //incoming real tx public key std::vector<crypto::public_key> real_out_additional_tx_keys; //incoming real tx additional public keys - size_t real_output_in_tx_index; //index in transaction outputs vector + uint64_t real_output_in_tx_index; //index in transaction outputs vector uint64_t amount; //money bool rct; //true if the output is rct rct::key mask; //ringct amount mask diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt index d28b44bb7..85c25546f 100644 --- a/src/cryptonote_protocol/CMakeLists.txt +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required (VERSION 2.6) +cmake_minimum_required (VERSION 3.5) project (monero CXX) file(GLOB CRYPTONOTE_PROTOCOL *) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 28530f3e7..80dd2bc39 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -46,6 +46,8 @@ #include "block_queue.h" #include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" +#include "net/levin_base.h" +#include "p2p/net_node_common.h" #include <boost/circular_buffer.hpp> PUSH_WARNINGS @@ -195,10 +197,11 @@ namespace cryptonote bool post_notify(typename t_parameter::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parameter).name() << " -->"); - epee::byte_slice blob; - epee::serialization::store_t_to_binary(arg, blob, 256 * 1024); // optimize for block responses + + epee::levin::message_writer out{256 * 1024}; // optimize for block responses + epee::serialization::store_t_to_binary(arg, out.buffer); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::to_span(blob), context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, std::move(out), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index c798dbcdb..685968c08 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -137,6 +137,41 @@ namespace cryptonote CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count); --context.m_callback_request_count; + uint32_t notified = true; + if (context.m_idle_peer_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) + { + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const auto ms = dt.total_microseconds(); + if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) + { + if (context.m_score-- >= 0) + { + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_expect_response = 0; + context.m_expect_height = 0; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + } + else + { + MINFO(context << "dropping idle peer with negative score"); + drop_connection_with_score(context, context.m_expect_response == 0 ? 1 : 5, false); + return false; + } + } + } + } + + notified = true; + if (context.m_new_stripe_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_normal) + context.m_state = cryptonote_connection_context::state_synchronizing; + } + if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time) { NOTIFY_REQUEST_CHAIN::request r = {}; @@ -1687,7 +1722,7 @@ skip: const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); if (stripe && peer_stripe && peer_stripe != stripe) return true; - context.m_state = cryptonote_connection_context::state_synchronizing; + context.m_new_stripe_notification = true; LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); @@ -1710,7 +1745,6 @@ skip: bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector<std::pair<boost::uuids::uuid, unsigned>> idle_peers; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) @@ -1720,36 +1754,16 @@ skip: const auto ms = dt.total_microseconds(); if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) { - if (context.m_score-- >= 0) - { - MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); - LOG_PRINT_CCONTEXT_L2("requesting callback"); - context.m_last_request_time = boost::date_time::not_a_date_time; - context.m_expect_response = 0; - context.m_expect_height = 0; - context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download - ++context.m_callback_request_count; - m_p2p->request_callback(context); - } - else - { - idle_peers.push_back(std::make_pair(context.m_connection_id, context.m_expect_response == 0 ? 1 : 5)); - } + context.m_idle_peer_notification = true; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); } } return true; }); - for (const auto &e: idle_peers) - { - const auto &uuid = e.first; - m_p2p->for_connection(uuid, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ - MINFO(ctx << "dropping idle peer with negative score"); - drop_connection_with_score(ctx, e.second, false); - return true; - }); - } - return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -2296,7 +2310,7 @@ skip: const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); const uint32_t first_stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); const uint32_t last_stripe = tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); - if ((((first_stripe && peer_stripe != first_stripe) || (last_stripe && peer_stripe != last_stripe)) && !m_sync_pruned_blocks) || (m_sync_pruned_blocks && req.prune)) + if (((first_stripe && peer_stripe != first_stripe) || (last_stripe && peer_stripe != last_stripe)) && !m_sync_pruned_blocks) { MDEBUG(context << "We need full data, but the peer does not have it, dropping peer"); return false; @@ -2699,15 +2713,15 @@ skip: // send fluffy ones first, we want to encourage people to run that if (!fluffyConnections.empty()) { - epee::byte_slice fluffyBlob; - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob, 32 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::to_span(fluffyBlob), std::move(fluffyConnections)); + epee::levin::message_writer fluffyBlob{32 * 1024}; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, std::move(fluffyBlob), std::move(fluffyConnections)); } if (!fullConnections.empty()) { - epee::byte_slice fullBlob; - epee::serialization::store_t_to_binary(arg, fullBlob, 128 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::to_span(fullBlob), std::move(fullConnections)); + epee::levin::message_writer fullBlob{128 * 1024}; + epee::serialization::store_t_to_binary(arg, fullBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, std::move(fullBlob), std::move(fullConnections)); } return true; @@ -2840,15 +2854,12 @@ skip: epee::string_tools::to_string_hex(context.m_pruning_seed) << "), score " << score << ", flush_all_spans " << flush_all_spans); - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + if (score > 0) + m_p2p->add_host_fail(context.m_remote_address, score); - // copy since dropping the connection will invalidate the context, and thus the address - const auto remote_address = context.m_remote_address; + m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); m_p2p->drop_connection(context); - - if (score > 0) - m_p2p->add_host_fail(remote_address, score); } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 79c2edf1d..57b1d049c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -30,8 +30,8 @@ #pragma once -#include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "cryptonote_protocol/enums.h" #include "cryptonote_basic/connection_context.h" namespace cryptonote { diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 1e9f3e399..0b065c3c3 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -159,7 +159,7 @@ namespace levin return get_out_connections(p2p, get_blockchain_height(p2p, core)); } - epee::byte_slice make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) + epee::levin::message_writer make_tx_message(std::vector<blobdata>&& txs, const bool pad, const bool fluff) { NOTIFY_NEW_TRANSACTIONS::request request{}; request.txs = std::move(txs); @@ -193,21 +193,17 @@ namespace levin // if the size of _ moved enough, we might lose byte in size encoding, we don't care } - epee::byte_slice fullBlob; - if (!epee::serialization::store_t_to_binary(request, fullBlob)) + epee::levin::message_writer out; + if (!epee::serialization::store_t_to_binary(request, out.buffer)) throw std::runtime_error{"Failed to serialize to epee binary format"}; - return fullBlob; + return out; } bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff) { - const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); - p2p.for_connection(destination, [&blob](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, blob.size(), NOTIFY_NEW_TRANSACTIONS::ID); - return true; - }); - return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); + epee::byte_slice blob = make_tx_message(std::move(txs), pad, fluff).finalize_notify(NOTIFY_NEW_TRANSACTIONS::ID); + return p2p.send(std::move(blob), destination); } /* The current design uses `asio::strand`s. The documentation isn't as clear @@ -653,10 +649,6 @@ namespace levin else message = zone_->noise.clone(); - zone_->p2p->for_connection(channel.connection, [&](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, message.size(), "noise"); - return true; - }); if (zone_->p2p->send(std::move(message), channel.connection)) { if (!channel.queue.empty() && channel.active.empty()) @@ -816,9 +808,8 @@ namespace levin // Padding is not useful when using noise mode. Send as stem so receiver // forwards in Dandelion++ mode. - const epee::byte_slice payload = make_tx_payload(std::move(txs), false, false); epee::byte_slice message = epee::levin::make_fragmented_notify( - zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(payload) + zone_->noise.size(), NOTIFY_NEW_TRANSACTIONS::ID, make_tx_message(std::move(txs), false, false) ); if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size()) { diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 5a7560874..14233bf29 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -30,6 +30,8 @@ #include "common/command_line.h" #include "net/parse.h" #include "daemon/command_parser_executor.h" +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -986,17 +988,67 @@ bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std:: bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::string>& args) { - const size_t args_count = args.size(); - if (args_count < 1 || args_count > 3) + struct parsed_t + { + std::string address; + std::string user; + std::string password; + std::string proxy; + }; + + boost::optional<parsed_t> parsed = [&args]() -> boost::optional<parsed_t> { + const size_t args_count = args.size(); + if (args_count == 0) + { + return {}; + } + if (args[0] == "auto") + { + if (args_count == 1) + { + return {{args[0], "", "", ""}}; + } + if (args_count == 2) + { + return {{args[0], "", "", args[1]}}; + } + } + else if (args[0] == "none") + { + if (args_count == 1) + { + return {{"", "", "", ""}}; + } + } + else + { + if (args_count == 1) + { + return {{args[0], "", "", ""}}; + } + if (args_count == 2) + { + return {{args[0], "", "", args[1]}}; + } + if (args_count == 3) + { + return {{args[0], args[1], args[2], ""}}; + } + if (args_count == 4) + { + return {{args[0], args[1], args[2], args[3]}}; + } + } + return {}; + }(); + + if (!parsed) { std::cout << "Invalid syntax: Wrong number of parameters. For more details, use the help command." << std::endl; return true; } - return m_executor.set_bootstrap_daemon( - args[0] != "none" ? args[0] : std::string(), - args_count > 1 ? args[1] : std::string(), - args_count > 2 ? args[2] : std::string()); + return m_executor.set_bootstrap_daemon(parsed->address, parsed->user, parsed->password, parsed->proxy); } bool t_command_parser_executor::flush_cache(const std::vector<std::string>& args) diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 4768bb842..63f44c4cd 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -326,7 +326,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "set_bootstrap_daemon" , std::bind(&t_command_parser_executor::set_bootstrap_daemon, &m_parser, p::_1) - , "set_bootstrap_daemon (auto | none | host[:port] [username] [password])" + , "set_bootstrap_daemon (auto | none | host[:port] [username] [password]) [proxy_ip:proxy_port]" , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced.\n" "Use 'auto' to enable automatic public nodes discovering and bootstrap daemon switching" ); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 8194fe642..16e6a304c 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -2405,7 +2405,8 @@ bool t_rpc_command_executor::check_blockchain_pruning() bool t_rpc_command_executor::set_bootstrap_daemon( const std::string &address, const std::string &username, - const std::string &password) + const std::string &password, + const std::string &proxy) { cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::request req; cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::response res; @@ -2414,6 +2415,7 @@ bool t_rpc_command_executor::set_bootstrap_daemon( req.address = address; req.username = username; req.password = password; + req.proxy = proxy; if (m_is_rpc) { diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 6fb5d6903..118f04731 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -168,7 +168,8 @@ public: bool set_bootstrap_daemon( const std::string &address, const std::string &username, - const std::string &password); + const std::string &password, + const std::string &proxy); bool rpc_payments(); diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dd4701e4f..c039b93c5 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -133,6 +133,18 @@ int main(int argc, char* argv[]) { std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; + bool parsed = cryptonote::parse_tx_extra(block.miner_tx.extra, fields); + if (!parsed) + std::cout << "Failed to parse tx_extra" << std::endl; + + if (!fields.empty()) + { + print_extra_fields(fields); + } + else + { + std::cout << "No fields were found in tx_extra" << std::endl; + } } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index ff2afba4b..3597ab336 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -71,6 +71,7 @@ target_link_libraries(device PUBLIC ${HIDAPI_LIBRARIES} cncrypto + cryptonote_format_utils_basic ringct_basic wallet-crypto ${OPENSSL_CRYPTO_LIBRARIES} diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 7c3e8785d..c2070b0d1 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -28,6 +28,8 @@ // #include "device_trezor.hpp" +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> namespace hw { namespace trezor { diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 70dc7f539..5f21ecd49 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -31,6 +31,7 @@ #include "memwipe.h" #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/split.hpp> +#include <boost/algorithm/string/predicate.hpp> #include <boost/regex.hpp> namespace hw { diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 288f3ddca..92150b579 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -502,21 +502,9 @@ namespace tx { } void Signer::compute_integrated_indices(TsxData * tsx_data){ - if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ - return; - } - auto & chg = tsx_data->change_dts(); std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); - std::vector<uint32_t> integrated_indices; - std::set<std::string> integrated_hashes; - for (auto & cur : m_aux_data->tx_recipients){ - if (!cur.has_payment_id){ - continue; - } - integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); - } ssize_t idx = -1; for (auto & cur : tsx_data->outputs()){ @@ -527,8 +515,7 @@ namespace tx { continue; } - c_hash = hash_addr(&cur.addr()); - if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + if (cur.is_integrated()){ integrated_indices.push_back((uint32_t)idx); } } diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index fa824ec3b..0fdd36a51 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -66,9 +66,7 @@ namespace protocol{ template<typename T> bool cn_deserialize(const void * buff, size_t len, T & dst){ - std::stringstream ss; - ss.write(static_cast<const char *>(buff), len); //ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{{reinterpret_cast<const std::uint8_t*>(buff), len}}; bool r = ::serialization::serialize(ba, dst); return r; } diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index a56090de0..881848a80 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -38,6 +38,7 @@ #include <boost/asio/ip/udp.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> #include <boost/format.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "common/apply_permutation.h" #include "transport.hpp" #include "messages/messages-common.pb.h" @@ -156,7 +157,11 @@ namespace trezor{ #define PROTO_HEADER_SIZE 6 static size_t message_size(const google::protobuf::Message &req){ - return static_cast<size_t>(req.ByteSize()); +#if GOOGLE_PROTOBUF_VERSION < 3006001 + return size_t(req.ByteSize()); +#else + return req.ByteSizeLong(); +#endif } static size_t serialize_message_buffer_size(size_t msg_size) { @@ -572,8 +577,13 @@ namespace trezor{ std::string req = "PINGPING"; char res[8]; - m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); - receive(res, 8, nullptr, false, timeout); + const auto written = m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); + if (written != req.size()) + return false; + memset(res, 0, sizeof(res)); + const auto received = receive(res, 8, nullptr, false, timeout); + if (received != 8) + return false; return memcmp(res, "PONGPONG", 8) == 0; diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index b6bc22a3d..8c79a53ca 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -491,6 +491,14 @@ namespace crypto return "<language not found>"; } + bool is_valid_language(const std::string &language) + { + const std::vector<const Language::Base*> language_instances = get_language_list(); + for (std::vector<const Language::Base*>::const_iterator it = language_instances.begin(); it != language_instances.end(); it++) + if ((*it)->get_english_language_name() == language || (*it)->get_language_name() == language) + return true; + return false; + } } } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 8d4c5be66..eb0c99e0b 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -124,6 +124,8 @@ namespace crypto * \return the name of the language in English */ std::string get_english_name_for(const std::string &name); + + bool is_valid_language(const std::string &language); } } diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index afcd42ef7..e93e27bcd 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,10 +26,10 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp socks.cpp - socks_connect.cpp tor_address.cpp zmq.cpp) -set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h socks_connect.h - tor_address.h zmq.h) +set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp resolve.cpp + socks.cpp socks_connect.cpp tor_address.cpp zmq.cpp) +set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h resolve.h + socks_connect.h tor_address.h zmq.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${ZMQ_LIB} ${Boost_ASIO_LIBRARY}) diff --git a/src/net/error.cpp b/src/net/error.cpp index 037f44d52..d2e713bc5 100644 --- a/src/net/error.cpp +++ b/src/net/error.cpp @@ -47,12 +47,18 @@ namespace { switch (net::error(value)) { + case net::error::bogus_dnssec: + return "Invalid response signature from DNSSEC enabled domain"; + case net::error::dns_query_failure: + return "Failed to retrieve desired DNS record"; case net::error::expected_tld: return "Expected top-level domain"; case net::error::invalid_host: return "Host value is not valid"; case net::error::invalid_i2p_address: return "Invalid I2P address"; + case net::error::invalid_mask: + return "CIDR netmask outside of 0-32 range"; case net::error::invalid_port: return "Invalid port value (expected 0-65535)"; case net::error::invalid_tor_address: @@ -71,6 +77,7 @@ namespace switch (net::error(value)) { case net::error::invalid_port: + case net::error::invalid_mask: return std::errc::result_out_of_range; case net::error::expected_tld: case net::error::invalid_tor_address: diff --git a/src/net/error.h b/src/net/error.h index 7c852dd20..746eb0ecb 100644 --- a/src/net/error.h +++ b/src/net/error.h @@ -37,13 +37,16 @@ namespace net enum class error : int { // 0 reserved for success (as per expect<T>) - expected_tld = 1, //!< Expected a tld + bogus_dnssec = 1, //!< Invalid response signature from DNSSEC enabled domain + dns_query_failure, //!< Failed to retrieve desired DNS record + expected_tld, //!< Expected a tld invalid_host, //!< Hostname is not valid invalid_i2p_address, + invalid_mask, //!< Outside of 0-32 range invalid_port, //!< Outside of 0-65535 range invalid_tor_address,//!< Invalid base32 or length unsupported_address,//!< Type not supported by `get_network_address` - invalid_mask, //!< Outside of 0-32 range + }; //! \return `std::error_category` for `net` namespace. diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp index 6c03b3808..ada4eb0d3 100644 --- a/src/net/i2p_address.cpp +++ b/src/net/i2p_address.cpp @@ -38,7 +38,7 @@ #include "net/error.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" -#include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/parse.cpp b/src/net/parse.cpp index 8a98e941a..298576ba4 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -31,6 +31,7 @@ #include "net/tor_address.h" #include "net/i2p_address.h" #include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/resolve.cpp b/src/net/resolve.cpp new file mode 100644 index 000000000..1b43cf6c4 --- /dev/null +++ b/src/net/resolve.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/resolve.h" + +#include <boost/utility/string_ref.hpp> +#include "common/dns_utils.h" +#include "common/expect.h" +#include "net/error.h" + +namespace net +{ +namespace dnssec +{ + expect<service_response> resolve_hostname(const std::string& addr, const std::string& tlsa_port) + { + // use basic (blocking) unbound for now, possibly refactor later + tools::DNSResolver& resolver = tools::DNSResolver::instance(); + + bool dnssec_available = false; + bool dnssec_valid = false; + std::vector<std::string> ip_records = resolver.get_ipv4(addr, dnssec_available, dnssec_valid); + + if (dnssec_available && !dnssec_valid) + return {net::error::bogus_dnssec}; + + if (ip_records.empty()) + { + ip_records = resolver.get_ipv6(addr, dnssec_available, dnssec_valid); + if (dnssec_available && !dnssec_valid) + return {net::error::bogus_dnssec}; + if (ip_records.empty()) + return {net::error::dns_query_failure}; + } + + std::vector<std::string> tlsa{}; + if (dnssec_available && !tlsa_port.empty()) + { + tlsa = resolver.get_tlsa_tcp_record(addr, tlsa_port, dnssec_available, dnssec_valid); + if (!dnssec_valid) + return {net::error::bogus_dnssec}; + } + return {{std::move(ip_records), std::move(tlsa)}}; + } +} // dnssec +} // net diff --git a/src/net/resolve.h b/src/net/resolve.h new file mode 100644 index 000000000..46bd8e617 --- /dev/null +++ b/src/net/resolve.h @@ -0,0 +1,47 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string> +#include <vector> + +template<typename> class expect; + +namespace net +{ +namespace dnssec +{ + struct service_response + { + std::vector<std::string> ip; //!< IPv4/6 records in dotted or semicolon notation + std::vector<std::string> tlsa; //!< DANE/TLSA records + }; + + //! \return IP + (optionally) DANE/TLSA records, failing if DNSSEC signature is "bogus" + expect<service_response> resolve_hostname(const std::string& addr, const std::string& tlsa_port = {}); +} // dnssec +} // net diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp index 9db9d4483..c797a24d4 100644 --- a/src/net/socks_connect.cpp +++ b/src/net/socks_connect.cpp @@ -38,6 +38,7 @@ #include "net/net_utils_base.h" #include "net/socks.h" #include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp index 4414861e7..a04dcb042 100644 --- a/src/net/tor_address.cpp +++ b/src/net/tor_address.cpp @@ -38,7 +38,7 @@ #include "net/error.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" -#include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index cfd8273b4..0cd38f253 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required (VERSION 2.6) +cmake_minimum_required (VERSION 3.5) project (monero CXX) file(GLOB P2P *) diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index db931122e..f2888674b 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -344,10 +344,9 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) final; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay); - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) final; virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 07b45b9bd..8dd4b4476 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -682,11 +682,15 @@ namespace nodetool full_addrs.insert("212.83.175.67:28080"); full_addrs.insert("212.83.172.165:28080"); full_addrs.insert("192.110.160.146:28080"); + full_addrs.insert("88.99.173.38:28080"); + full_addrs.insert("51.79.173.165:28080"); } else if (m_nettype == cryptonote::STAGENET) { full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("192.110.160.146:38080"); + full_addrs.insert("88.99.173.38:38080"); + full_addrs.insert("51.79.173.165:38080"); } else if (m_nettype == cryptonote::FAKECHAIN) { @@ -701,6 +705,8 @@ namespace nodetool full_addrs.insert("209.250.243.248:18080"); full_addrs.insert("104.238.221.81:18080"); full_addrs.insert("66.85.74.134:18080"); + full_addrs.insert("88.99.173.38:18080"); + full_addrs.insert("51.79.173.165:18080"); } return full_addrs; } @@ -828,6 +834,7 @@ namespace nodetool "xwvz3ekocr3dkyxfkmgm2hvbpzx2ysqmaxgter7znnqrhoicygkfswid.onion:18083", "4pixvbejrvihnkxmduo2agsnmc3rrulrqc7s3cbwwrep6h6hrzsibeqd.onion:18083", "zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083", + "qz43zul2x56jexzoqgkx2trzwcfnr6l3hbtfcfx54g4r3eahy3bssjyd.onion:18083", }; } return {}; @@ -1203,9 +1210,8 @@ namespace nodetool if(!handle_remote_peerlist(rsp.local_peerlist_new, context)) { LOG_WARNING_CC(context, "COMMAND_TIMED_SYNC: failed to handle_remote_peerlist(...), closing connection."); - const auto remote_address = context.m_remote_address; m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); - add_host_fail(remote_address); + add_host_fail(context.m_remote_address); } if(!context.m_is_income) m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash); @@ -1230,8 +1236,8 @@ namespace nodetool if(!max_index) return 0; - size_t x = crypto::rand<size_t>()%(max_index+1); - size_t res = (x*x*x)/(max_index*max_index); //parabola \/ + size_t x = crypto::rand<size_t>()%(16*max_index+1); + size_t res = (x*x*x)/(max_index*max_index*16*16*16); //parabola \/ MDEBUG("Random connection index=" << res << "(x="<< x << ", max_index=" << max_index << ")"); return res; } @@ -1369,7 +1375,7 @@ namespace nodetool if(just_take_peerlist) { zone.m_net_server.get_config_object().close(con->m_connection_id); - MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -1431,7 +1437,7 @@ namespace nodetool zone.m_net_server.get_config_object().close(con->m_connection_id); - MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -2003,15 +2009,22 @@ namespace nodetool { if (ip.empty()) continue; + auto subnet = net::get_ipv4_subnet_address(ip); + if (subnet) + { + block_subnet(*subnet, DNS_BLOCKLIST_LIFETIME); + ++good; + continue; + } const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(ip, 0); - if (!parsed_addr) + if (parsed_addr) { - MWARNING("Invalid IP address from DNS blocklist: " << ip << " - " << parsed_addr.error()); - ++bad; + block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true); + ++good; continue; } - block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true); - ++good; + MWARNING("Invalid IP address or subnet from DNS blocklist: " << ip << " - " << parsed_addr.error()); + ++bad; } } if (good > 0) @@ -2169,8 +2182,9 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, epee::levin::message_writer data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { + epee::byte_slice message = data_buff.finalize_notify(command); std::sort(connections.begin(), connections.end()); auto zone = m_network_zones.begin(); for(const auto& c_id: connections) @@ -2188,7 +2202,7 @@ namespace nodetool ++zone; } if (zone->first == c_id.first) - zone->second.m_net_server.get_config_object().notify(command, data_buff, c_id.second); + zone->second.m_net_server.get_config_object().send(message.clone(), c_id.second); } return true; } @@ -2255,24 +2269,13 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(const int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { if(is_filtered_command(context.m_remote_address, command)) return false; network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); - return res > 0; - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - if(is_filtered_command(context.m_remote_address, command)) - return false; - - network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); + int res = zone.m_net_server.get_config_object().send(message.finalize_notify(command), context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- @@ -2458,14 +2461,12 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_handshake(int command, typename COMMAND_HANDSHAKE::request& arg, typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { - // copy since dropping the connection will invalidate the context, and thus the address - const auto remote_address = context.m_remote_address; - if(arg.node_data.network_id != m_network_id) { + LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); - add_host_fail(remote_address); + add_host_fail(context.m_remote_address); return 1; } @@ -2473,7 +2474,7 @@ namespace nodetool { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came not from incoming connection"); drop_connection(context); - add_host_fail(remote_address); + add_host_fail(context.m_remote_address); return 1; } diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 0da758ad4..92b7596ae 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -40,6 +40,8 @@ #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" +namespace epee { namespace levin { class message_writer; } } + namespace nodetool { @@ -49,10 +51,9 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay)=0; - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_public_connections_count()=0; @@ -71,7 +72,7 @@ namespace nodetool template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { return false; } @@ -79,11 +80,7 @@ namespace nodetool { return epee::net_utils::zone::invalid; } - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - return false; - } - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { return true; } diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index c794b0f3b..d8de6abe9 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -110,7 +110,7 @@ namespace nodetool bool get_gray_peer_by_index(peerlist_entry& p, size_t i); template<typename F> bool foreach(bool white, const F &f); void evict_host_from_white_peerlist(const peerlist_entry& pr); - bool append_with_peer_white(const peerlist_entry& pr); + bool append_with_peer_white(const peerlist_entry& pr, bool trust_last_seen = false); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port, uint32_t rpc_credits_per_hash); @@ -329,12 +329,12 @@ namespace nodetool ple.pruning_seed = pruning_seed; ple.rpc_port = rpc_port; ple.rpc_credits_per_hash = rpc_credits_per_hash; - return append_with_peer_white(ple); + return append_with_peer_white(ple, true); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::append_with_peer_white(const peerlist_entry& ple) + bool peerlist_manager::append_with_peer_white(const peerlist_entry& ple, bool trust_last_seen) { TRY_ENTRY(); if(!is_host_allowed(ple.adr)) @@ -357,7 +357,8 @@ namespace nodetool new_ple.pruning_seed = by_addr_it_wt->pruning_seed; if (by_addr_it_wt->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around new_ple.rpc_port = by_addr_it_wt->rpc_port; - new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + if (!trust_last_seen) + new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer lists are untrusted m_peers_white.replace(by_addr_it_wt, new_ple); } //remove from gray list, if need diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 00b72123a..278ff4164 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -284,7 +284,7 @@ namespace rct { { FIELD(type) if (type == RCTTypeNull) - return ar.stream().good(); + return ar.good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2 && type != RCTTypeCLSAG) return false; VARINT_FIELD(txnFee) @@ -344,7 +344,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - return ar.stream().good(); + return ar.good(); } BEGIN_SERIALIZE_OBJECT() @@ -375,7 +375,7 @@ namespace rct { if (mixin >= 0xffffffff) return false; if (type == RCTTypeNull) - return ar.stream().good(); + return ar.good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2 && type != RCTTypeCLSAG) return false; if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2 || type == RCTTypeCLSAG) @@ -522,7 +522,7 @@ namespace rct { } ar.end_array(); } - return ar.stream().good(); + return ar.good(); } BEGIN_SERIALIZE_OBJECT() diff --git a/src/rpc/bootstrap_daemon.cpp b/src/rpc/bootstrap_daemon.cpp index 2fdd28406..ffea906d5 100644 --- a/src/rpc/bootstrap_daemon.cpp +++ b/src/rpc/bootstrap_daemon.cpp @@ -7,6 +7,7 @@ #include "crypto/crypto.h" #include "cryptonote_core/cryptonote_core.h" #include "misc_log_ex.h" +#include "net/parse.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc.bootstrap_daemon" @@ -16,19 +17,23 @@ namespace cryptonote bootstrap_daemon::bootstrap_daemon( std::function<std::map<std::string, bool>()> get_public_nodes, - bool rpc_payment_enabled) + bool rpc_payment_enabled, + const std::string &proxy) : m_selector(new bootstrap_node::selector_auto(std::move(get_public_nodes))) , m_rpc_payment_enabled(rpc_payment_enabled) { + set_proxy(proxy); } bootstrap_daemon::bootstrap_daemon( const std::string &address, boost::optional<epee::net_utils::http::login> credentials, - bool rpc_payment_enabled) + bool rpc_payment_enabled, + const std::string &proxy) : m_selector(nullptr) , m_rpc_payment_enabled(rpc_payment_enabled) { + set_proxy(proxy); if (!set_server(address, std::move(credentials))) { throw std::runtime_error("invalid bootstrap daemon address or credentials"); @@ -78,6 +83,18 @@ namespace cryptonote return success; } + void bootstrap_daemon::set_proxy(const std::string &address) + { + if (!address.empty() && !net::get_tcp_endpoint(address)) + { + throw std::runtime_error("invalid proxy address format"); + } + if (!m_http_client.set_proxy(address)) + { + throw std::runtime_error("failed to set proxy address"); + } + } + bool bootstrap_daemon::set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials /* = boost::none */) { if (!m_http_client.set_server(address, credentials)) diff --git a/src/rpc/bootstrap_daemon.h b/src/rpc/bootstrap_daemon.h index d54042b11..1e4477123 100644 --- a/src/rpc/bootstrap_daemon.h +++ b/src/rpc/bootstrap_daemon.h @@ -8,7 +8,7 @@ #include <boost/thread/mutex.hpp> #include <boost/utility/string_ref.hpp> -#include "net/http_client.h" +#include "net/http.h" #include "storages/http_abstract_invoke.h" #include "bootstrap_node_selector.h" @@ -21,11 +21,13 @@ namespace cryptonote public: bootstrap_daemon( std::function<std::map<std::string, bool>()> get_public_nodes, - bool rpc_payment_enabled); + bool rpc_payment_enabled, + const std::string &proxy); bootstrap_daemon( const std::string &address, boost::optional<epee::net_utils::http::login> credentials, - bool rpc_payment_enabled); + bool rpc_payment_enabled, + const std::string &proxy); std::string address() const noexcept; boost::optional<std::pair<uint64_t, uint64_t>> get_height(); @@ -72,12 +74,14 @@ namespace cryptonote return handle_result(result, result_struct.status); } + void set_proxy(const std::string &address); + private: bool set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials = boost::none); bool switch_server_if_needed(); private: - epee::net_utils::http::http_simple_client m_http_client; + net::http::client m_http_client; const bool m_rpc_payment_enabled; const std::unique_ptr<bootstrap_node::selector> m_selector; boost::mutex m_selector_mutex; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 2bb4969e9..8d8a68efb 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -30,6 +30,7 @@ #include <boost/preprocessor/stringize.hpp> #include <boost/uuid/nil_generator.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -44,6 +45,7 @@ using namespace epee; #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_basic/merge_mining.h" #include "cryptonote_core/tx_sanity_check.h" #include "misc_language.h" #include "net/parse.h" @@ -154,6 +156,7 @@ namespace cryptonote command_line::add_arg(desc, arg_restricted_rpc); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); + command_line::add_arg(desc, arg_bootstrap_daemon_proxy); cryptonote::rpc_args::init_options(desc, true); command_line::add_arg(desc, arg_rpc_payment_address); command_line::add_arg(desc, arg_rpc_payment_difficulty); @@ -172,7 +175,10 @@ namespace cryptonote , m_rpc_payment_allow_free_loopback(false) {} //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const std::string &username_password) + bool core_rpc_server::set_bootstrap_daemon( + const std::string &address, + const std::string &username_password, + const std::string &proxy) { boost::optional<epee::net_utils::http::login> credentials; const auto loc = username_password.find(':'); @@ -180,7 +186,7 @@ namespace cryptonote { credentials = epee::net_utils::http::login(username_password.substr(0, loc), username_password.substr(loc + 1)); } - return set_bootstrap_daemon(address, credentials); + return set_bootstrap_daemon(address, credentials, proxy); } //------------------------------------------------------------------------------------------------------------------------------ std::map<std::string, bool> core_rpc_server::get_public_nodes(uint32_t credits_per_hash_threshold/* = 0*/) @@ -217,7 +223,10 @@ namespace cryptonote return result; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials) + bool core_rpc_server::set_bootstrap_daemon( + const std::string &address, + const boost::optional<epee::net_utils::http::login> &credentials, + const std::string &proxy) { boost::unique_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); @@ -233,11 +242,11 @@ namespace cryptonote auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; - m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled)); + m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, proxy)); } else { - m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled)); + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, proxy)); } m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; @@ -278,6 +287,7 @@ namespace cryptonote } } disable_rpc_ban = rpc_config->disable_rpc_ban; + const std::string data_dir{command_line::get_arg(vm, cryptonote::arg_data_dir)}; std::string address = command_line::get_arg(vm, arg_rpc_payment_address); if (!address.empty() && allow_rpc_payment) { @@ -306,7 +316,7 @@ namespace cryptonote } m_rpc_payment_allow_free_loopback = command_line::get_arg(vm, arg_rpc_payment_allow_free_loopback); m_rpc_payment.reset(new rpc_payment(info.address, diff, credits)); - m_rpc_payment->load(command_line::get_arg(vm, cryptonote::arg_data_dir)); + m_rpc_payment->load(data_dir); m_p2p.set_rpc_credits_per_hash(RPC_CREDITS_PER_HASH_SCALE * (credits / (float)diff)); } @@ -318,8 +328,10 @@ namespace cryptonote MWARNING("The RPC server is accessible from the outside, but no RPC payment was setup. RPC access will be free for all."); } - if (!set_bootstrap_daemon(command_line::get_arg(vm, arg_bootstrap_daemon_address), - command_line::get_arg(vm, arg_bootstrap_daemon_login))) + if (!set_bootstrap_daemon( + command_line::get_arg(vm, arg_bootstrap_daemon_address), + command_line::get_arg(vm, arg_bootstrap_daemon_login), + command_line::get_arg(vm, arg_bootstrap_daemon_proxy))) { MFATAL("Failed to parse bootstrap daemon address"); return false; @@ -333,12 +345,32 @@ namespace cryptonote if (m_rpc_payment) m_net_server.add_idle_handler([this](){ return m_rpc_payment->on_idle(); }, 60 * 1000); + bool store_ssl_key = !restricted && rpc_config->ssl_options && rpc_config->ssl_options.auth.certificate_path.empty(); + const auto ssl_base_path = (boost::filesystem::path{data_dir} / "rpc_ssl").string(); + if (store_ssl_key && boost::filesystem::exists(ssl_base_path + ".crt")) + { + // load key from previous run, password prompted by OpenSSL + store_ssl_key = false; + rpc_config->ssl_options.auth = + epee::net_utils::ssl_authentication_t{ssl_base_path + ".key", ssl_base_path + ".crt"}; + } + auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; - return epee::http_server_impl_base<core_rpc_server, connection_context>::init( + const bool inited = epee::http_server_impl_base<core_rpc_server, connection_context>::init( rng, std::move(port), std::move(bind_ip_str), std::move(bind_ipv6_str), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); + + if (store_ssl_key && inited) + { + // new keys were generated, store for next run + const auto error = epee::net_utils::store_ssl_keys(m_net_server.get_ssl_context(), ssl_base_path); + if (error) + MFATAL("Failed to store HTTP SSL cert/key for " << (restricted ? "restricted " : "") << "RPC server: " << error.message()); + return !bool(error); + } + return inited; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::check_payment(const std::string &client_message, uint64_t payment, const std::string &rpc, bool same_ts, std::string &message, uint64_t &credits, std::string &top_hash) @@ -573,7 +605,7 @@ namespace cryptonote if (max_blocks == 0) { res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; - return false; + return true; } } @@ -582,7 +614,7 @@ namespace cryptonote { res.status = "Failed"; add_host_fail(ctx); - return false; + return true; } CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); @@ -618,12 +650,12 @@ namespace cryptonote if (!r) { res.status = "Failed"; - return false; + return true; } if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) { res.status = "Failed"; - return false; + return true; } for (size_t i = 0; i < indices.size(); ++i) res.output_indices.back().indices.push_back({std::move(indices[i])}); @@ -646,7 +678,7 @@ namespace cryptonote if(!m_core.get_alternative_blocks(blks)) { res.status = "Failed"; - return false; + return true; } res.blks_hashes.reserve(blks.size()); @@ -717,7 +749,7 @@ namespace cryptonote { res.status = "Failed"; add_host_fail(ctx); - return false; + return true; } CHECK_PAYMENT_SAME_TS(req, res, res.m_block_ids.size() * COST_PER_BLOCK_HASH); @@ -1012,6 +1044,7 @@ namespace cryptonote if (e.in_pool) { e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max(); + e.confirmations = 0; auto it = per_tx_pool_tx_info.find(tx_hash); if (it != per_tx_pool_tx_info.end()) { @@ -1030,6 +1063,7 @@ namespace cryptonote else { e.block_height = m_core.get_blockchain_storage().get_db().get_tx_block_height(tx_hash); + e.confirmations = m_core.get_current_blockchain_height() - e.block_height; e.block_timestamp = m_core.get_blockchain_storage().get_db().get_block_timestamp(e.block_height); e.received_timestamp = 0; e.double_spend_seen = false; @@ -1048,7 +1082,7 @@ namespace cryptonote if (!r) { res.status = "Failed"; - return false; + return true; } } } @@ -1424,7 +1458,8 @@ namespace cryptonote res.status = peer_list_res.status; if (!success) { - return false; + res.status = "Failed to get peer list"; + return true; } if (res.status != CORE_RPC_STATUS_OK) { @@ -1595,15 +1630,15 @@ namespace cryptonote { credentials = epee::net_utils::http::login(req.username, req.password); } - - if (set_bootstrap_daemon(req.address, credentials)) + + if (set_bootstrap_daemon(req.address, credentials, req.proxy)) { res.status = CORE_RPC_STATUS_OK; } else { res.status = "Failed to set bootstrap daemon"; - } + } return true; } @@ -1626,7 +1661,7 @@ namespace cryptonote if (m_should_use_bootstrap_daemon) { res.status = "This command is unsupported for bootstrap daemon"; - return false; + return true; } } res.count = m_core.get_current_blockchain_height(); @@ -1642,7 +1677,7 @@ namespace cryptonote if (m_should_use_bootstrap_daemon) { res = "This command is unsupported for bootstrap daemon"; - return false; + return true; } } if(req.size() != 1) @@ -1826,6 +1861,125 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + RPC_TRACKER(add_aux_pow); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ADD_AUX_POW>(invoke_http_mode::JON_RPC, "add_aux_pow", req, res, r)) + return r; + + if (req.aux_pow.empty()) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Empty aux pow hash vector"; + return false; + } + + crypto::hash merkle_root; + size_t merkle_tree_depth = 0; + std::vector<std::pair<crypto::hash, crypto::hash>> aux_pow; + std::vector<crypto::hash> aux_pow_raw; + aux_pow.reserve(req.aux_pow.size()); + aux_pow_raw.reserve(req.aux_pow.size()); + for (const auto &s: req.aux_pow) + { + aux_pow.push_back({}); + if (!epee::string_tools::hex_to_pod(s.id, aux_pow.back().first)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid aux pow id"; + return false; + } + if (!epee::string_tools::hex_to_pod(s.hash, aux_pow.back().second)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid aux pow hash"; + return false; + } + aux_pow_raw.push_back(aux_pow.back().second); + } + + size_t path_domain = 1; + while ((1u << path_domain) < aux_pow.size()) + ++path_domain; + uint32_t nonce; + const uint32_t max_nonce = 65535; + bool collision = true; + for (nonce = 0; nonce <= max_nonce; ++nonce) + { + std::vector<bool> slots(aux_pow.size(), false); + collision = false; + for (size_t idx = 0; idx < aux_pow.size(); ++idx) + { + const uint32_t slot = cryptonote::get_aux_slot(aux_pow[idx].first, nonce, aux_pow.size()); + if (slot >= aux_pow.size()) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Computed slot is out of range"; + return false; + } + if (slots[slot]) + { + collision = true; + break; + } + slots[slot] = true; + } + if (!collision) + break; + } + if (collision) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to find a suitable nonce"; + return false; + } + + crypto::tree_hash((const char(*)[crypto::HASH_SIZE])aux_pow_raw.data(), aux_pow_raw.size(), merkle_root.data); + res.merkle_root = epee::string_tools::pod_to_hex(merkle_root); + res.merkle_tree_depth = cryptonote::encode_mm_depth(aux_pow.size(), nonce); + + blobdata blocktemplate_blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.blocktemplate_blob, blocktemplate_blob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid blocktemplate_blob"; + return false; + } + + block b; + if (!parse_and_validate_block_from_blob(blocktemplate_blob, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong blocktemplate_blob"; + return false; + } + + if (!remove_field_from_tx_extra(b.miner_tx.extra, typeid(cryptonote::tx_extra_merge_mining_tag))) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Error removing existing merkle root"; + return false; + } + if (!add_mm_merkle_root_to_tx_extra(b.miner_tx.extra, merkle_root, merkle_tree_depth)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Error adding merkle root"; + return false; + } + b.invalidate_hashes(); + b.miner_tx.invalidate_hashes(); + + const blobdata block_blob = t_serializable_object_to_blob(b); + const blobdata hashing_blob = get_block_hashing_blob(b); + + res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); + res.blockhashing_blob = string_tools::buff_to_hex_nodelimer(hashing_blob); + res.aux_pow = req.aux_pow; + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(submitblock); @@ -1833,7 +1987,8 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); if (m_should_use_bootstrap_daemon) { - res.status = "This command is unsupported for bootstrap daemon"; + error_resp.code = CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP; + error_resp.message = "This command is unsupported for bootstrap daemon"; return false; } } @@ -2525,7 +2680,7 @@ namespace cryptonote if (!m_core.get_blockchain_storage().flush_txes_from_pool(txids)) { res.status = "Failed to remove one or more tx(es)"; - return false; + return true; } if (failed) @@ -2534,7 +2689,7 @@ namespace cryptonote res.status = "Failed to parse txid"; else res.status = "Failed to parse some of the txids"; - return false; + return true; } res.status = CORE_RPC_STATUS_OK; @@ -2693,7 +2848,7 @@ namespace cryptonote if (req.limit_down != -1) { res.status = CORE_RPC_ERROR_CODE_WRONG_PARAM; - return false; + return true; } epee::net_utils::connection_basic::set_rate_down_limit(nodetool::default_limit_down); } @@ -2707,7 +2862,7 @@ namespace cryptonote if (req.limit_up != -1) { res.status = CORE_RPC_ERROR_CODE_WRONG_PARAM; - return false; + return true; } epee::net_utils::connection_basic::set_rate_up_limit(nodetool::default_limit_up); } @@ -2811,17 +2966,17 @@ namespace cryptonote if (!tools::download(path.string(), res.auto_uri)) { MERROR("Failed to download " << res.auto_uri); - return false; + return true; } if (!tools::sha256sum(path.string(), file_hash)) { MERROR("Failed to hash " << path); - return false; + return true; } if (hash != epee::string_tools::pod_to_hex(file_hash)) { MERROR("Download from " << res.auto_uri << " does not match the expected hash"); - return false; + return true; } MINFO("New version downloaded to " << path); } @@ -2896,6 +3051,8 @@ namespace cryptonote if (failed) { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = res.status; return false; } @@ -3007,7 +3164,7 @@ namespace cryptonote if (!req.binary) { res.status = "Binary only call"; - return false; + return true; } try { @@ -3019,7 +3176,7 @@ namespace cryptonote if (!data) { res.status = "Failed to get output distribution"; - return false; + return true; } res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); @@ -3028,7 +3185,7 @@ namespace cryptonote catch (const std::exception &e) { res.status = "Failed to get output distribution"; - return false; + return true; } res.status = CORE_RPC_STATUS_OK; @@ -3262,7 +3419,8 @@ namespace cryptonote if (!m_rpc_payment) { - res.status = "Payments not enabled"; + error_resp.code = CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED; + error_resp.message = "Payments not enabled"; return false; } @@ -3290,7 +3448,8 @@ namespace cryptonote if (!m_rpc_payment) { - res.status = "Payments not enabled"; + error_resp.code = CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED; + error_resp.message = "Payments not enabled"; return false; } @@ -3347,6 +3506,12 @@ namespace cryptonote , "" }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_proxy = { + "bootstrap-daemon-proxy" + , "<ip>:<port> socks proxy to use for bootstrap daemon connections" + , "" + }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_payment_address = { "rpc-payment-address" , "Restrict RPC to clients sending micropayment to this address" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index dcf6b4e4b..b21e43ab0 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -72,6 +72,7 @@ namespace cryptonote static const command_line::arg_descriptor<bool> arg_rpc_ssl_allow_any_cert; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_address; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_login; + static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_proxy; static const command_line::arg_descriptor<std::string> arg_rpc_payment_address; static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_difficulty; static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_credits; @@ -146,6 +147,7 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE_IF("generateblocks", on_generateblocks, COMMAND_RPC_GENERATEBLOCKS, !m_restricted) @@ -226,6 +228,7 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); @@ -268,8 +271,14 @@ private: uint64_t get_block_reward(const block& blk); bool fill_block_header_response(const block& blk, bool orphan_status, uint64_t height, const crypto::hash& hash, block_header_response& response, bool fill_pow_hash); std::map<std::string, bool> get_public_nodes(uint32_t credits_per_hash_threshold = 0); - bool set_bootstrap_daemon(const std::string &address, const std::string &username_password); - bool set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials); + bool set_bootstrap_daemon( + const std::string &address, + const std::string &username_password, + const std::string &proxy); + bool set_bootstrap_daemon( + const std::string &address, + const boost::optional<epee::net_utils::http::login> &credentials, + const std::string &proxy); enum invoke_http_mode { JON, BIN, JON_RPC }; template <typename COMMAND_TYPE> bool use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index bbcb27f1c..ff8c98b98 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 5 +#define CORE_RPC_VERSION_MINOR 7 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -350,6 +350,7 @@ namespace cryptonote bool in_pool; bool double_spend_seen; uint64_t block_height; + uint64_t confirmations; uint64_t block_timestamp; uint64_t received_timestamp; std::vector<uint64_t> output_indices; @@ -367,6 +368,7 @@ namespace cryptonote if (!this_ref.in_pool) { KV_SERIALIZE(block_height) + KV_SERIALIZE(confirmations) KV_SERIALIZE(block_timestamp) KV_SERIALIZE(output_indices) } @@ -938,6 +940,52 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_ADD_AUX_POW + { + struct aux_pow_t + { + std::string id; + std::string hash; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(hash) + END_KV_SERIALIZE_MAP() + }; + + struct request_t: public rpc_request_base + { + blobdata blocktemplate_blob; + std::vector<aux_pow_t> aux_pow; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(aux_pow) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + blobdata blocktemplate_blob; + blobdata blockhashing_blob; + std::string merkle_root; + uint32_t merkle_tree_depth; + std::vector<aux_pow_t> aux_pow; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(blockhashing_blob) + KV_SERIALIZE(merkle_root) + KV_SERIALIZE(merkle_tree_depth) + KV_SERIALIZE(aux_pow) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_SUBMITBLOCK { typedef std::vector<std::string> request; @@ -1613,11 +1661,13 @@ namespace cryptonote std::string address; std::string username; std::string password; + std::string proxy; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) KV_SERIALIZE(username) KV_SERIALIZE(password) + KV_SERIALIZE(proxy) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 1458049ab..232df0373 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -49,6 +49,8 @@ #define CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT -17 #define CORE_RPC_ERROR_CODE_STALE_PAYMENT -18 #define CORE_RPC_ERROR_CODE_RESTRICTED -19 +#define CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP -20 +#define CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED -21 static inline const char *get_rpc_server_error_message(int64_t code) { @@ -72,6 +74,8 @@ static inline const char *get_rpc_server_error_message(int64_t code) case CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT: return "Duplicate payment"; case CORE_RPC_ERROR_CODE_STALE_PAYMENT: return "Stale payment"; case CORE_RPC_ERROR_CODE_RESTRICTED: return "Parameters beyond restricted allowance"; + case CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP: return "Command is unsupported in bootstrap mode"; + case CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED: return "Payments not enabled"; default: MERROR("Unknown error: " << code); return "Unknown error"; } } diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 176f11fa3..f0c513911 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/filesystem.hpp> #include "cryptonote_config.h" #include "include_base_utils.h" #include "string_tools.h" @@ -292,12 +293,13 @@ namespace cryptonote MINFO("loading rpc payments data from " << state_file_path); std::ifstream data; data.open(state_file_path, std::ios_base::binary | std::ios_base::in); + std::string bytes(std::istream_iterator<char>{data}, std::istream_iterator<char>{}); if (!data.fail()) { bool loaded = false; try { - binary_archive<false> ar(data); + binary_archive<false> ar{epee::strspan<std::uint8_t>(bytes)}; if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -305,6 +307,8 @@ namespace cryptonote catch (...) {} if (!loaded) { + bytes.clear(); + bytes.shrink_to_fit(); try { boost::archive::portable_binary_iarchive a(data); diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 80104c8f7..acda70039 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -36,9 +36,11 @@ #include <cassert> #include <iostream> #include <iterator> +#include <boost/endian/conversion.hpp> #include <boost/type_traits/make_unsigned.hpp> #include "common/varint.h" +#include "span.h" #include "warnings.h" /* I have no clue what these lines means */ @@ -55,16 +57,15 @@ DISABLE_VS_WARNINGS(4244) * purpse is to define the functions used for the binary_archive. Its * a header, basically. I think it was declared simply to save typing... */ -template <class Stream, bool IsSaving> +template <bool IsSaving> struct binary_archive_base { - typedef Stream stream_type; - typedef binary_archive_base<Stream, IsSaving> base_type; + typedef binary_archive_base<IsSaving> base_type; typedef boost::mpl::bool_<IsSaving> is_saving; typedef uint8_t variant_tag_type; - explicit binary_archive_base(stream_type &s) : stream_(s) { } + explicit binary_archive_base() { } /* definition of standard API functions */ void tag(const char *) { } @@ -72,12 +73,6 @@ struct binary_archive_base void end_object() { } void begin_variant() { } void end_variant() { } - /* I just want to leave a comment saying how this line really shows - flaws in the ownership model of many OOP languages, that is all. */ - stream_type &stream() { return stream_; } - -protected: - stream_type &stream_; }; /* \struct binary_archive @@ -95,15 +90,18 @@ struct binary_archive; template <> -struct binary_archive<false> : public binary_archive_base<std::istream, false> +struct binary_archive<false> : public binary_archive_base<false> { + explicit binary_archive(epee::span<const std::uint8_t> s) + : base_type(), bytes_(s), begin_(s.begin()), good_(true), varint_bug_backward_compatibility_(false) + {} - explicit binary_archive(stream_type &s) : base_type(s) { - stream_type::pos_type pos = stream_.tellg(); - stream_.seekg(0, std::ios_base::end); - eof_pos_ = stream_.tellg(); - stream_.seekg(pos); - } + bool good() const noexcept { return good_; } + void set_fail() noexcept { good_ = false; } + + //! If implementing as `std::istream`, reset stream error state after `peek()` call. + bool eof() const noexcept { return bytes_.empty(); } + std::size_t getpos() const noexcept { return bytes_.begin() - begin_; } template <class T> void serialize_int(T &v) @@ -116,24 +114,24 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> * \brief serializes an unsigned integer */ template <class T> - void serialize_uint(T &v, size_t width = sizeof(T)) + void serialize_uint(T &v) { - T ret = 0; - unsigned shift = 0; - for (size_t i = 0; i < width; i++) { - //std::cerr << "tell: " << stream_.tellg() << " value: " << ret << std::endl; - char c; - stream_.get(c); - T b = (unsigned char)c; - ret += (b << shift); // can this be changed to OR, i think it can. - shift += 8; + const std::size_t actual = bytes_.remove_prefix(sizeof(T)); + good_ &= (actual == sizeof(T)); + if (actual == sizeof(T)) + { + std::memcpy(std::addressof(v), bytes_.data() - sizeof(T), sizeof(T)); + boost::endian::little_to_native_inplace(v); // epee isn't templated } - v = ret; + else + v = 0; // ensures initialization } void serialize_blob(void *buf, size_t len, const char *delimiter="") { - stream_.read((char *)buf, len); + const std::size_t actual = bytes_.remove_prefix(len); + good_ &= (len == actual); + std::memcpy(buf, bytes_.data() - actual, actual); } template <class T> @@ -145,9 +143,11 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> template <class T> void serialize_uvarint(T &v) { - typedef std::istreambuf_iterator<char> it; - if (tools::read_varint(it(stream_), it(), v) < 0) - stream_.setstate(std::ios_base::failbit); + auto current = bytes_.cbegin(); + auto end = bytes_.cend(); + good_ &= (0 <= tools::read_varint(current, end, v)); + current = std::min(current, bytes_.cend()); + bytes_ = {current, std::size_t(bytes_.cend() - current)}; } void begin_array(size_t &s) @@ -166,21 +166,26 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> serialize_int(t); } - size_t remaining_bytes() { - if (!stream_.good()) - return 0; - //std::cerr << "tell: " << stream_.tellg() << std::endl; - assert(stream_.tellg() <= eof_pos_); - return eof_pos_ - stream_.tellg(); - } + size_t remaining_bytes() const noexcept { return good() ? bytes_.size() : 0; } + void enable_varint_bug_backward_compatibility() { varint_bug_backward_compatibility_ = true; } + bool varint_bug_backward_compatibility_enabled() const { return varint_bug_backward_compatibility_; } protected: - std::streamoff eof_pos_; + epee::span<const std::uint8_t> bytes_; + std::uint8_t const* const begin_; + bool good_; + bool varint_bug_backward_compatibility_; }; template <> -struct binary_archive<true> : public binary_archive_base<std::ostream, true> +struct binary_archive<true> : public binary_archive_base<true> { - explicit binary_archive(stream_type &s) : base_type(s) { } + typedef std::ostream stream_type; + explicit binary_archive(stream_type &s) : base_type(), stream_(s) { } + + bool good() const { return stream_.good(); } + void set_fail() { stream_.setstate(std::ios::failbit); } + + std::streampos getpos() const { return stream_.tellp(); } template <class T> void serialize_int(T v) @@ -227,6 +232,10 @@ struct binary_archive<true> : public binary_archive_base<std::ostream, true> void write_variant_tag(variant_tag_type t) { serialize_int(t); } + + bool varint_bug_backward_compatibility_enabled() const { return false; } +protected: + stream_type& stream_; }; POP_WARNINGS diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index 8635585af..8444f220e 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -39,8 +39,7 @@ namespace serialization { template <class T> bool parse_binary(const std::string &blob, T &v) { - std::istringstream istr(blob); - binary_archive<false> iar(istr); + binary_archive<false> iar{epee::strspan<std::uint8_t>(blob)}; return ::serialization::serialize(iar, v); } diff --git a/src/serialization/container.h b/src/serialization/container.h index d5e75bb4f..77681ec93 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -32,22 +32,27 @@ namespace serialization { namespace detail { - template <typename Archive, class T> - bool serialize_container_element(Archive& ar, T& e) + template<typename T> + inline constexpr bool use_container_varint() noexcept { - return ::do_serialize(ar, e); + return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1; } - template <typename Archive> - bool serialize_container_element(Archive& ar, uint32_t& e) + template <typename Archive, class T> + typename std::enable_if<!use_container_varint<T>(), bool>::type + serialize_container_element(Archive& ar, T& e) { - ar.serialize_varint(e); - return true; + return ::do_serialize(ar, e); } - template <typename Archive> - bool serialize_container_element(Archive& ar, uint64_t& e) + template<typename Archive, typename T> + typename std::enable_if<use_container_varint<T>(), bool>::type + serialize_container_element(Archive& ar, T& e) { + static constexpr const bool previously_varint = std::is_same<uint64_t, T>() || std::is_same<uint32_t, T>(); + + if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) + return ::do_serialize(ar, e); ar.serialize_varint(e); return true; } @@ -62,13 +67,13 @@ bool do_serialize_container(Archive<false> &ar, C &v) { size_t cnt; ar.begin_array(cnt); - if (!ar.stream().good()) + if (!ar.good()) return false; v.clear(); // very basic sanity check if (ar.remaining_bytes() < cnt) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } @@ -81,7 +86,7 @@ bool do_serialize_container(Archive<false> &ar, C &v) if (!::serialization::detail::serialize_container_element(ar, e)) return false; ::serialization::detail::do_add(v, std::move(e)); - if (!ar.stream().good()) + if (!ar.good()) return false; } ar.end_array(); @@ -95,13 +100,13 @@ bool do_serialize_container(Archive<true> &ar, C &v) ar.begin_array(cnt); for (auto i = v.begin(); i != v.end(); ++i) { - if (!ar.stream().good()) + if (!ar.good()) return false; if (i != v.begin()) ar.delimit_array(); if(!::serialization::detail::serialize_container_element(ar, (typename C::value_type&)*i)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; } ar.end_array(); diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index 42bd06e92..d635a9ee9 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -47,7 +47,7 @@ bool do_serialize(Archive<false> &ar, std::vector<crypto::signature> &v) // very basic sanity check if (ar.remaining_bytes() < cnt*sizeof(crypto::signature)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } @@ -55,7 +55,7 @@ bool do_serialize(Archive<false> &ar, std::vector<crypto::signature> &v) for (size_t i = 0; i < cnt; i++) { v.resize(i+1); ar.serialize_blob(&(v[i]), sizeof(crypto::signature), ""); - if (!ar.stream().good()) + if (!ar.good()) return false; } return true; @@ -70,7 +70,7 @@ bool do_serialize(Archive<true> &ar, std::vector<crypto::signature> &v) size_t cnt = v.size(); for (size_t i = 0; i < cnt; i++) { ar.serialize_blob(&(v[i]), sizeof(crypto::signature), ""); - if (!ar.stream().good()) + if (!ar.good()) return false; } ar.end_string(); diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index b04d6ae19..e8ccb9a58 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -38,6 +38,7 @@ struct debug_archive : public json_archive<W> { typedef typename json_archive<W>::stream_type stream_type; debug_archive(stream_type &s) : json_archive<W>(s) { } + stream_type& stream() { return this->stream_; } }; template <class T> diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h index 56c0312e7..75d1fd13f 100644 --- a/src/serialization/difficulty_type.h +++ b/src/serialization/difficulty_type.h @@ -38,10 +38,10 @@ inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) { uint64_t hi, lo; ar.serialize_varint(hi); - if (!ar.stream().good()) + if (!ar.good()) return false; ar.serialize_varint(lo); - if (!ar.stream().good()) + if (!ar.good()) return false; diff = hi; diff <<= 64; @@ -52,13 +52,13 @@ inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) template <template <bool> class Archive> inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff) { - if (!ar.stream().good()) + if (!ar.good()) return false; const uint64_t hi = ((diff >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); const uint64_t lo = (diff & 0xffffffffffffffff).convert_to<uint64_t>(); ar.serialize_varint(hi); ar.serialize_varint(lo); - if (!ar.stream().good()) + if (!ar.good()) return false; return true; } diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index 50dd5bbd0..bab6dfa94 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -58,6 +58,10 @@ struct json_archive_base json_archive_base(stream_type &s, bool indent = false) : stream_(s), indent_(indent), object_begin(false), depth_(0) { } + bool good() const { return stream_.good(); } + void set_fail() { stream_.setstate(std::ios::failbit); } + void clear_fail() { stream_.clear(); } + void tag(const char *tag) { if (!object_begin) stream_ << ", "; @@ -82,7 +86,8 @@ struct json_archive_base void begin_variant() { begin_object(); } void end_variant() { end_object(); } - Stream &stream() { return stream_; } + + bool varint_bug_backward_compatibility_enabled() const { return false; } protected: void make_indent() @@ -115,6 +120,8 @@ struct json_archive<true> : public json_archive_base<std::ostream, true> { json_archive(stream_type &s, bool indent = false) : base_type(s, indent), inner_array_size_(0) { } + std::streampos getpos() const { return stream_.tellp(); } + template<typename T> static auto promote_to_printable_integer_type(T v) -> decltype(+v) { diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index de14c8911..35ea990b3 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -365,7 +365,7 @@ inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type t static_assert(!std::is_same<value_type, unsigned char>::value, "encoding an array of unsigned char is faster as hex"); dest.StartArray(); - for (const auto& t : vec) + for (auto t : vec) toJsonValue(dest, t); dest.EndArray(); } diff --git a/src/serialization/pair.h b/src/serialization/pair.h index 18280d837..2d9a89242 100644 --- a/src/serialization/pair.h +++ b/src/serialization/pair.h @@ -30,21 +30,34 @@ #pragma once #include <memory> +#include <boost/type_traits/make_unsigned.hpp> #include "serialization.h" namespace serialization { namespace detail { + template<typename T> + inline constexpr bool use_pair_varint() noexcept + { + return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1; + } + template <typename Archive, class T> - bool serialize_pair_element(Archive& ar, T& e) + typename std::enable_if<!use_pair_varint<T>(), bool>::type + serialize_pair_element(Archive& ar, T& e) { return ::do_serialize(ar, e); } - template <typename Archive> - bool serialize_pair_element(Archive& ar, uint64_t& e) + template<typename Archive, typename T> + typename std::enable_if<use_pair_varint<T>(), bool>::type + serialize_pair_element(Archive& ar, T& e) { + static constexpr const bool previously_varint = std::is_same<uint64_t, T>(); + + if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) + return ::do_serialize(ar, e); ar.serialize_varint(e); return true; } @@ -56,19 +69,19 @@ inline bool do_serialize(Archive<false>& ar, std::pair<F,S>& p) { size_t cnt; ar.begin_array(cnt); - if (!ar.stream().good()) + if (!ar.good()) return false; if (cnt != 2) return false; if (!::serialization::detail::serialize_pair_element(ar, p.first)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.delimit_array(); if (!::serialization::detail::serialize_pair_element(ar, p.second)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.end_array(); @@ -79,16 +92,16 @@ template <template <bool> class Archive, class F, class S> inline bool do_serialize(Archive<true>& ar, std::pair<F,S>& p) { ar.begin_array(2); - if (!ar.stream().good()) + if (!ar.good()) return false; if(!::serialization::detail::serialize_pair_element(ar, p.first)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.delimit_array(); if(!::serialization::detail::serialize_pair_element(ar, p.second)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.end_array(); return true; diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 3ebeed171..4acadeab3 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -213,7 +213,7 @@ inline bool do_serialize(Archive &ar, bool &v) * \brief self-explanatory */ #define END_SERIALIZE() \ - return ar.stream().good(); \ + return ar.good(); \ } /*! \macro VALUE(f) @@ -223,7 +223,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(#f); \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro FIELD_N(t,f) @@ -234,7 +234,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(t); \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro FIELD(f) @@ -245,7 +245,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(#f); \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro FIELDS(f) @@ -255,7 +255,7 @@ inline bool do_serialize(Archive &ar, bool &v) #define FIELDS(f) \ do { \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro VARINT_FIELD(f) @@ -265,7 +265,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(#f); \ ar.serialize_varint(f); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ } while(0); /*! \macro VARINT_FIELD_N(t, f) @@ -276,7 +276,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(t); \ ar.serialize_varint(f); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ } while(0); /*! \macro MAGIC_FIELD(m) @@ -286,7 +286,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag("magic"); \ ar.serialize_blob((void*)magic.data(), magic.size()); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ if (magic != m) return false; \ } while(0); @@ -297,7 +297,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag("version"); \ ar.serialize_varint(version); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ } while(0); @@ -339,10 +339,10 @@ namespace serialization { * * \brief self explanatory */ - template<class Stream> - bool do_check_stream_state(Stream& s, boost::mpl::bool_<true>, bool noeof) + template<class Archive> + bool do_check_stream_state(Archive& ar, boost::mpl::bool_<true>, bool noeof) { - return s.good(); + return ar.good(); } /*! \fn do_check_stream_state * @@ -350,15 +350,13 @@ namespace serialization { * * \detailed Also checks to make sure that the stream is not at EOF */ - template<class Stream> - bool do_check_stream_state(Stream& s, boost::mpl::bool_<false>, bool noeof) + template<class Archive> + bool do_check_stream_state(Archive& ar, boost::mpl::bool_<false>, bool noeof) { bool result = false; - if (s.good()) + if (ar.good()) { - std::ios_base::iostate state = s.rdstate(); - result = noeof || EOF == s.peek(); - s.clear(state); + result = noeof || ar.eof(); } return result; } @@ -371,7 +369,7 @@ namespace serialization { template<class Archive> bool check_stream_state(Archive& ar, bool noeof = false) { - return detail::do_check_stream_state(ar.stream(), typename Archive::is_saving(), noeof); + return detail::do_check_stream_state(ar, typename Archive::is_saving(), noeof); } /*! \fn serialize diff --git a/src/serialization/string.h b/src/serialization/string.h index f5f69833a..f1f8f4ab0 100644 --- a/src/serialization/string.h +++ b/src/serialization/string.h @@ -39,7 +39,7 @@ inline bool do_serialize(Archive<false>& ar, std::string& str) ar.serialize_varint(size); if (ar.remaining_bytes() < size) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } diff --git a/src/serialization/variant.h b/src/serialization/variant.h index b08cf6bf0..6debb63d1 100644 --- a/src/serialization/variant.h +++ b/src/serialization/variant.h @@ -74,7 +74,7 @@ struct variant_reader current_type x; if(!::do_serialize(ar, x)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } v = x; @@ -95,7 +95,7 @@ struct variant_reader<Archive, Variant, TBegin, TBegin> static inline bool read(Archive &ar, Variant &v, variant_tag_type t) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } }; @@ -116,7 +116,7 @@ struct serializer<Archive<false>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> typename boost::mpl::begin<types>::type, typename boost::mpl::end<types>::type>::read(ar, v, t)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } ar.end_variant(); @@ -143,7 +143,7 @@ struct serializer<Archive<true>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> ar.write_variant_tag(variant_serialization_traits<Archive<true>, T>::get_tag()); if(!::do_serialize(ar, rv)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } ar.end_variant(); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 2a3c33f48..dc031b36c 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -50,6 +50,7 @@ #include <boost/format.hpp> #include <boost/regex.hpp> #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "console_handler.h" #include "common/i18n.h" @@ -276,7 +277,7 @@ namespace const char* USAGE_PUBLIC_NODES("public_nodes"); const char* USAGE_WELCOME("welcome"); const char* USAGE_RPC_PAYMENT_INFO("rpc_payment_info"); - const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc"); + const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc [<number_of_threads>]"); const char* USAGE_STOP_MINING_FOR_RPC("stop_mining_for_rpc"); const char* USAGE_SHOW_QR_CODE("show_qr_code [<subaddress_index>]"); const char* USAGE_VERSION("version"); @@ -607,9 +608,14 @@ void simple_wallet::handle_transfer_exception(const std::exception_ptr &e, bool fail_msg_writer() << e.what(); warn_of_possible_attack = false; } + catch (const tools::error::zero_amount&) + { + fail_msg_writer() << sw::tr("destination amount is zero"); + warn_of_possible_attack = false; + } catch (const tools::error::zero_destination&) { - fail_msg_writer() << sw::tr("one of destinations is zero"); + fail_msg_writer() << sw::tr("transaction has no destination"); warn_of_possible_attack = false; } catch (const tools::error::tx_too_big& e) @@ -2361,6 +2367,24 @@ bool simple_wallet::start_mining_for_rpc(const std::vector<std::string> &args) if (!try_connect_to_daemon()) return true; + bool ok = true; + if(args.size() >= 1) + { + uint16_t num = 0; + ok = string_tools::get_xtype_from_string(num, args[0]); + m_rpc_payment_threads = num; + } + else + { + m_rpc_payment_threads = 0; + } + + if (!ok) + { + PRINT_USAGE(USAGE_START_MINING_FOR_RPC); + return true; + } + LOCK_IDLE_SCOPE(); bool payment_required; @@ -5871,7 +5895,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo if (reset != ResetNone) { if (reset == ResetSoftKeepKI) - height_pre = m_wallet->hash_m_transfers(-1, transfer_hash_pre); + height_pre = m_wallet->hash_m_transfers(boost::none, transfer_hash_pre); m_wallet->rescan_blockchain(reset == ResetHard, false, reset == ResetSoftKeepKI); } @@ -8898,12 +8922,12 @@ bool simple_wallet::export_transfers(const std::vector<std::string>& args_) // header file << - boost::format("%8.8s,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,%s,%s") % + boost::format("%8.8s,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%106.106s,%20.20s,%s,%s") % tr("block") % tr("direction") % tr("unlocked") % tr("timestamp") % tr("amount") % tr("running balance") % tr("hash") % tr("payment ID") % tr("fee") % tr("destination") % tr("amount") % tr("index") % tr("note") << std::endl; uint64_t running_balance = 0; - auto formatter = boost::format("%8.8llu,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,\"%s\",%s"); + auto formatter = boost::format("%8.8llu,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%106.106s,%20.20s,\"%s\",%s"); for (const auto& transfer : all_transfers) { @@ -9327,7 +9351,7 @@ bool simple_wallet::check_rpc_payment() fail_msg_writer() << tr("Error mining to daemon: ") << error; m_cmd_binder.print_prompt(); }; - bool ret = m_wallet->search_for_rpc_payment(target, startfunc, contfunc, foundfunc, errorfunc); + bool ret = m_wallet->search_for_rpc_payment(target, m_rpc_payment_threads, startfunc, contfunc, foundfunc, errorfunc); if (!ret) { fail_msg_writer() << tr("Failed to start mining for RPC payment"); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index af654e0dd..8780bee1d 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -464,6 +464,7 @@ namespace cryptonote std::atomic<bool> m_need_payment; boost::posix_time::ptime m_last_rpc_payment_mining_time; bool m_rpc_payment_mining_requested; + uint32_t m_rpc_payment_threads = 0; bool m_daemon_rpc_payment_message_displayed; float m_rpc_payment_hash_rate; std::atomic<bool> m_suspend_rpc_payment_mining; diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index bf238ae37..2dd64a38f 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -40,18 +40,7 @@ set(wallet_sources wallet_rpc_payments.cpp ) -set(wallet_private_headers - wallet2.h - wallet_args.h - wallet_errors.h - wallet_rpc_server.h - wallet_rpc_server_commands_defs.h - wallet_rpc_server_error_codes.h - ringdb.h - node_rpc_proxy.h - message_store.h - message_transporter.h - wallet_rpc_helpers.h) +monero_find_all_headers(wallet_private_headers "${CMAKE_CURRENT_SOURCE_DIR}") monero_private_headers(wallet ${wallet_private_headers}) @@ -115,45 +104,4 @@ if(NOT IOS) install(TARGETS wallet_rpc_server DESTINATION bin) endif() -# build and install libwallet_merged only if we building for GUI -if (BUILD_GUI_DEPS) - set(libs_to_merge - wallet_api - wallet - rpc_base - multisig - blockchain_db - cryptonote_core - cryptonote_basic - mnemonics - common - cncrypto - device - hardforks - ringct - ringct_basic - checkpoints - version - net - device_trezor) - - foreach(lib ${libs_to_merge}) - list(APPEND objlibs $<TARGET_OBJECTS:obj_${lib}>) # matches naming convention in src/CMakeLists.txt - endforeach() - add_library(wallet_merged STATIC ${objlibs}) - if(IOS) - set(lib_folder lib-${ARCH}) - else() - set(lib_folder lib) - endif() - install(TARGETS wallet_merged - ARCHIVE DESTINATION ${lib_folder}) - - install(FILES ${TREZOR_DEP_LIBS} - DESTINATION ${lib_folder}) - file(WRITE "trezor_link_flags.txt" ${TREZOR_DEP_LINKER}) - install(FILES "trezor_link_flags.txt" - DESTINATION ${lib_folder}) -endif() - add_subdirectory(api) diff --git a/src/wallet/api/CMakeLists.txt b/src/wallet/api/CMakeLists.txt index 30eb4ce03..655cdfefd 100644 --- a/src/wallet/api/CMakeLists.txt +++ b/src/wallet/api/CMakeLists.txt @@ -71,10 +71,13 @@ target_link_libraries(wallet_api mnemonics ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} + ${Boost_LOCALE_LIBRARY} + ${ICU_LIBRARIES} ${Boost_SERIALIZATION_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} ${Boost_REGEX_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) diff --git a/src/wallet/api/address_book.cpp b/src/wallet/api/address_book.cpp index 96090d9f5..0b0e17464 100644 --- a/src/wallet/api/address_book.cpp +++ b/src/wallet/api/address_book.cpp @@ -81,7 +81,7 @@ bool AddressBookImpl::setDescription(std::size_t index, const std::string &descr tools::wallet2::address_book_row entry = ab[index]; entry.m_description = description; - bool r = m_wallet->m_wallet->set_address_book_row(index, entry.m_address, NULL, entry.m_description, entry.m_is_subaddress); + bool r = m_wallet->m_wallet->set_address_book_row(index, entry.m_address, entry.m_has_payment_id ? &entry.m_payment_id : nullptr, entry.m_description, entry.m_is_subaddress); if (r) refresh(); else @@ -166,5 +166,3 @@ AddressBookImpl::~AddressBookImpl() } } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/address_book.h b/src/wallet/api/address_book.h index e22f474fb..40da46853 100644 --- a/src/wallet/api/address_book.h +++ b/src/wallet/api/address_book.h @@ -66,6 +66,3 @@ private: }; } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index 24f6d37db..f7e74591f 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -40,6 +40,7 @@ #include <vector> #include <sstream> #include <boost/format.hpp> +#include <boost/filesystem.hpp> using namespace std; @@ -263,6 +264,3 @@ std::vector<std::string> PendingTransactionImpl::signersKeys() const { } } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index bd0ca80a6..274c60851 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -73,5 +73,3 @@ private: } - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/transaction_history.cpp b/src/wallet/api/transaction_history.cpp index d8e4aab65..4649089ea 100644 --- a/src/wallet/api/transaction_history.cpp +++ b/src/wallet/api/transaction_history.cpp @@ -266,5 +266,3 @@ void TransactionHistoryImpl::refresh() } } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/transaction_history.h b/src/wallet/api/transaction_history.h index 60f12d771..fe77253e6 100644 --- a/src/wallet/api/transaction_history.h +++ b/src/wallet/api/transaction_history.h @@ -56,6 +56,3 @@ private: }; } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/transaction_info.cpp b/src/wallet/api/transaction_info.cpp index 33e7856db..edbdc469a 100644 --- a/src/wallet/api/transaction_info.cpp +++ b/src/wallet/api/transaction_info.cpp @@ -150,5 +150,3 @@ uint64_t TransactionInfoImpl::unlockTime() const } } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/transaction_info.h b/src/wallet/api/transaction_info.h index 8bc36a8e9..5eeeb04c2 100644 --- a/src/wallet/api/transaction_info.h +++ b/src/wallet/api/transaction_info.h @@ -87,5 +87,3 @@ private: }; } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/unsigned_transaction.cpp b/src/wallet/api/unsigned_transaction.cpp index fb96674a7..4ccfafebd 100644 --- a/src/wallet/api/unsigned_transaction.cpp +++ b/src/wallet/api/unsigned_transaction.cpp @@ -316,6 +316,3 @@ uint64_t UnsignedTransactionImpl::minMixinCount() const } } // namespace - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/unsigned_transaction.h b/src/wallet/api/unsigned_transaction.h index 81fc098ff..07649e39e 100644 --- a/src/wallet/api/unsigned_transaction.h +++ b/src/wallet/api/unsigned_transaction.h @@ -71,5 +71,3 @@ private: } - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/utils.cpp b/src/wallet/api/utils.cpp index 34debee9d..19151b5f6 100644 --- a/src/wallet/api/utils.cpp +++ b/src/wallet/api/utils.cpp @@ -60,5 +60,3 @@ void onStartup() } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 4f923ce54..b4fa1ea8c 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -45,10 +45,8 @@ #include <sstream> #include <unordered_map> -#ifdef WIN32 #include <boost/locale.hpp> #include <boost/filesystem.hpp> -#endif using namespace std; using namespace cryptonote; @@ -60,7 +58,6 @@ namespace Monero { namespace { // copy-pasted from simplewallet - static const size_t DEFAULT_MIXIN = 6; static const int DEFAULT_REFRESH_INTERVAL_MILLIS = 1000 * 10; // limit maximum refresh interval as one minute static const int MAX_REFRESH_INTERVAL_MILLIS = 1000 * 60 * 1; @@ -791,11 +788,11 @@ bool WalletImpl::close(bool store) return result; } -std::string WalletImpl::seed() const +std::string WalletImpl::seed(const std::string& seed_offset) const { epee::wipeable_string seed; if (m_wallet) - m_wallet->get_seed(seed); + m_wallet->get_seed(seed, seed_offset); return std::string(seed.data(), seed.size()); // TODO } @@ -839,6 +836,11 @@ bool WalletImpl::setPassword(const std::string &password) return status() == Status_Ok; } +const std::string& WalletImpl::getPassword() const +{ + return m_password; +} + bool WalletImpl::setDevicePin(const std::string &pin) { clearStatus(); @@ -1178,7 +1180,7 @@ bool WalletImpl::exportKeyImages(const string &filename, bool all) try { - if (!m_wallet->export_key_images(filename), all) + if (!m_wallet->export_key_images(filename, all)) { setStatusError(tr("failed to save file ") + filename); return false; @@ -1216,6 +1218,68 @@ bool WalletImpl::importKeyImages(const string &filename) return true; } +bool WalletImpl::exportOutputs(const string &filename, bool all) +{ + if (m_wallet->key_on_device()) + { + setStatusError(string(tr("Not supported on HW wallets.")) + filename); + return false; + } + + try + { + std::string data = m_wallet->export_outputs_to_str(all); + bool r = m_wallet->save_to_file(filename, data); + if (!r) + { + LOG_ERROR("Failed to save file " << filename); + setStatusError(string(tr("Failed to save file: ")) + filename); + return false; + } + } + catch (const std::exception &e) + { + LOG_ERROR("Error exporting outputs: " << e.what()); + setStatusError(string(tr("Error exporting outputs: ")) + e.what()); + return false; + } + + LOG_PRINT_L2("Outputs exported to " << filename); + return true; +} + +bool WalletImpl::importOutputs(const string &filename) +{ + if (m_wallet->key_on_device()) + { + setStatusError(string(tr("Not supported on HW wallets.")) + filename); + return false; + } + + std::string data; + bool r = m_wallet->load_from_file(filename, data); + if (!r) + { + LOG_ERROR("Failed to read file: " << filename); + setStatusError(string(tr("Failed to read file: ")) + filename); + return false; + } + + try + { + size_t n_outputs = m_wallet->import_outputs_from_str(data); + LOG_PRINT_L2(std::to_string(n_outputs) << " outputs imported"); + } + catch (const std::exception &e) + { + LOG_ERROR("Failed to import outputs: " << e.what()); + setStatusError(string(tr("Failed to import outputs: ")) + e.what()); + return false; + } + + return true; +} + void WalletImpl::addSubaddressAccount(const std::string& label) { m_wallet->add_subaddress_account(label); @@ -1431,13 +1495,6 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri cryptonote::address_parse_info info; - // indicates if dst_addr is integrated address (address + payment_id) - // TODO: (https://bitcointalk.org/index.php?topic=753252.msg9985441#msg9985441) - size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); - if (fake_outs_count == 0) - fake_outs_count = DEFAULT_MIXIN; - fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); - uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority)); PendingTransactionImpl * transaction = new PendingTransactionImpl(*this); @@ -1503,6 +1560,9 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri break; } try { + size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); + fake_outs_count = m_wallet->adjust_mixin(mixin_count); + if (amount) { transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, @@ -1567,8 +1627,10 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri setStatusError(writer.str()); } catch (const tools::error::tx_sum_overflow& e) { setStatusError(e.what()); + } catch (const tools::error::zero_amount&) { + setStatusError(tr("destination amount is zero")); } catch (const tools::error::zero_destination&) { - setStatusError(tr("one of destinations is zero")); + setStatusError(tr("transaction has no destination")); } catch (const tools::error::tx_too_big& e) { setStatusError(tr("failed to find a suitable way to split transactions")); } catch (const tools::error::transfer_error& e) { @@ -2001,9 +2063,24 @@ bool WalletImpl::checkReserveProof(const std::string &address, const std::string } } -std::string WalletImpl::signMessage(const std::string &message) +std::string WalletImpl::signMessage(const std::string &message, const std::string &address) { - return m_wallet->sign(message, tools::wallet2::sign_with_spend_key); + if (address.empty()) { + return m_wallet->sign(message, tools::wallet2::sign_with_spend_key); + } + + cryptonote::address_parse_info info; + if (!cryptonote::get_account_address_from_str(info, m_wallet->nettype(), address)) { + setStatusError(tr("Failed to parse address")); + return ""; + } + auto index = m_wallet->get_subaddress_index(info.address); + if (!index) { + setStatusError(tr("Address doesn't belong to the wallet")); + return ""; + } + + return m_wallet->sign(message, tools::wallet2::sign_with_spend_key, *index); } bool WalletImpl::verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const @@ -2104,6 +2181,11 @@ bool WalletImpl::watchOnly() const return m_wallet->watch_only(); } +bool WalletImpl::isDeterministic() const +{ + return m_wallet->is_deterministic(); +} + void WalletImpl::clearStatus() const { boost::lock_guard<boost::mutex> l(m_statusMutex); @@ -2305,6 +2387,10 @@ bool WalletImpl::rescanSpent() return true; } +void WalletImpl::setOffline(bool offline) +{ + m_wallet->set_offline(offline); +} void WalletImpl::hardForkInfo(uint8_t &version, uint64_t &earliest_height) const { @@ -2505,6 +2591,32 @@ void WalletImpl::deviceShowAddress(uint32_t accountIndex, uint32_t addressIndex, m_wallet->device_show_address(accountIndex, addressIndex, payment_id_param); } -} // namespace -namespace Bitmonero = Monero; +bool WalletImpl::reconnectDevice() +{ + clearStatus(); + + bool r; + try { + r = m_wallet->reconnect_device(); + } + catch (const std::exception &e) { + LOG_ERROR(__FUNCTION__ << " error: " << e.what()); + setStatusError(e.what()); + return false; + } + + return r; +} + +uint64_t WalletImpl::getBytesReceived() +{ + return m_wallet->get_bytes_received(); +} + +uint64_t WalletImpl::getBytesSent() +{ + return m_wallet->get_bytes_sent(); +} + +} // namespace diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index e501d3943..217f16e48 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -81,7 +81,7 @@ public: const std::string &device_name); Device getDeviceType() const override; bool close(bool store = true); - std::string seed() const override; + std::string seed(const std::string& seed_offset = "") const override; std::string getSeedLanguage() const override; void setSeedLanguage(const std::string &arg) override; // void setListener(Listener *) {} @@ -89,6 +89,7 @@ public: std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; bool setPassword(const std::string &password) override; + const std::string& getPassword() const override; bool setDevicePin(const std::string &password) override; bool setDevicePassphrase(const std::string &password) override; std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; @@ -129,6 +130,7 @@ public: void setRecoveringFromDevice(bool recoveringFromDevice) override; void setSubaddressLookahead(uint32_t major, uint32_t minor) override; bool watchOnly() const override; + bool isDeterministic() const override; bool rescanSpent() override; NetworkType nettype() const override {return static_cast<NetworkType>(m_wallet->nettype());} void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const override; @@ -166,6 +168,8 @@ public: virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename) override; bool exportKeyImages(const std::string &filename, bool all = false) override; bool importKeyImages(const std::string &filename) override; + bool exportOutputs(const std::string &filename, bool all = false) override; + bool importOutputs(const std::string &filename) override; virtual void disposeTransaction(PendingTransaction * t) override; virtual uint64_t estimateTransactionFee(const std::vector<std::pair<std::string, uint64_t>> &destinations, @@ -181,6 +185,8 @@ public: virtual bool setCacheAttribute(const std::string &key, const std::string &val) override; virtual std::string getCacheAttribute(const std::string &key) const override; + virtual void setOffline(bool offline) override; + virtual bool setUserNote(const std::string &txid, const std::string ¬e) override; virtual std::string getUserNote(const std::string &txid) const override; virtual std::string getTxKey(const std::string &txid) const override; @@ -191,7 +197,7 @@ public: virtual bool checkSpendProof(const std::string &txid, const std::string &message, const std::string &signature, bool &good) const override; virtual std::string getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const override; virtual bool checkReserveProof(const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &total, uint64_t &spent) const override; - virtual std::string signMessage(const std::string &message) override; + virtual std::string signMessage(const std::string &message, const std::string &address) override; virtual bool verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const override; virtual std::string signMultisigParticipant(const std::string &message) const override; virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const override; @@ -215,6 +221,9 @@ public: virtual bool isKeysFileLocked() override; virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) override; virtual void deviceShowAddress(uint32_t accountIndex, uint32_t addressIndex, const std::string &paymentId) override; + virtual bool reconnectDevice() override; + virtual uint64_t getBytesReceived() override; + virtual uint64_t getBytesSent() override; private: void clearStatus() const; @@ -276,7 +285,4 @@ private: } // namespace -namespace Bitmonero = Monero; - #endif - diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index b40b6763f..a08033033 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -446,7 +446,7 @@ struct Wallet }; virtual ~Wallet() = 0; - virtual std::string seed() const = 0; + virtual std::string seed(const std::string& seed_offset = "") const = 0; virtual std::string getSeedLanguage() const = 0; virtual void setSeedLanguage(const std::string &arg) = 0; //! returns wallet status (Status_Ok | Status_Error) @@ -456,6 +456,7 @@ struct Wallet //! returns both error and error string atomically. suggested to use in instead of status() and errorString() virtual void statusWithErrorString(int& status, std::string& errorString) const = 0; virtual bool setPassword(const std::string &password) = 0; + virtual const std::string& getPassword() const = 0; virtual bool setDevicePin(const std::string &pin) { (void)pin; return false; }; virtual bool setDevicePassphrase(const std::string &passphrase) { (void)passphrase; return false; }; virtual std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const = 0; @@ -627,6 +628,12 @@ struct Wallet virtual bool watchOnly() const = 0; /** + * @brief isDeterministic - checks if wallet keys are deterministic + * @return - true if deterministic + */ + virtual bool isDeterministic() const = 0; + + /** * @brief blockChainHeight - returns current blockchain height * @return */ @@ -913,6 +920,19 @@ struct Wallet */ virtual bool importKeyImages(const std::string &filename) = 0; + /*! + * \brief importOutputs - exports outputs to file + * \param filename + * \return - true on success + */ + virtual bool exportOutputs(const std::string &filename, bool all = false) = 0; + + /*! + * \brief importOutputs - imports outputs from file + * \param filename + * \return - true on success + */ + virtual bool importOutputs(const std::string &filename) = 0; virtual TransactionHistory * history() = 0; virtual AddressBook * addressBook() = 0; @@ -974,7 +994,7 @@ struct Wallet * \param message - the message to sign (arbitrary byte data) * \return the signature */ - virtual std::string signMessage(const std::string &message) = 0; + virtual std::string signMessage(const std::string &message, const std::string &address = "") = 0; /*! * \brief verifySignedMessage - verify a signature matches a given message * \param message - the message (arbitrary byte data) @@ -1008,6 +1028,12 @@ struct Wallet * \return true on success */ virtual bool rescanSpent() = 0; + + /* + * \brief setOffline - toggle set offline on/off + * \param offline - true/false + */ + virtual void setOffline(bool offline) = 0; //! blackballs a set of outputs virtual bool blackballOutputs(const std::vector<std::string> &outputs, bool add) = 0; @@ -1059,6 +1085,15 @@ struct Wallet //! shows address on device display virtual void deviceShowAddress(uint32_t accountIndex, uint32_t addressIndex, const std::string &paymentId) = 0; + + //! attempt to reconnect to hardware device + virtual bool reconnectDevice() = 0; + + //! get bytes received + virtual uint64_t getBytesReceived() = 0; + + //! get bytes sent + virtual uint64_t getBytesSent() = 0; }; /** @@ -1337,6 +1372,3 @@ struct WalletManagerFactory } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 900fe91e5..417a27db5 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -406,5 +406,3 @@ void WalletManagerFactory::setLogCategories(const std::string &categories) } - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index 2f603b0a9..cf3056a17 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -102,5 +102,3 @@ private: }; } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 87cb75fbf..34b4f440b 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -30,6 +30,8 @@ #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/format.hpp> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <fstream> #include <sstream> #include "file_io_utils.h" @@ -191,9 +193,7 @@ void message_store::unpack_signer_config(const multisig_wallet_state &state, con { try { - std::stringstream iss; - iss << signer_config; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(signer_config)}; THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, signers), tools::error::wallet_internal_error, "Failed to serialize signer config"); } catch (...) @@ -381,9 +381,7 @@ void message_store::process_auto_config_data_message(uint32_t id) auto_config_data data; try { - std::stringstream iss; - iss << m.content; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(m.content)}; THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, data), tools::error::wallet_internal_error, "Failed to serialize auto config data"); } catch (...) @@ -788,9 +786,7 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std file_data read_file_data; try { - std::stringstream iss; - iss << buf; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(buf)}; if (::serialization::serialize(ar, read_file_data)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -827,9 +823,7 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std loaded = false; try { - std::stringstream iss; - iss << decrypted_data; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(decrypted_data)}; if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 1c4671150..133db6c2a 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -37,6 +37,7 @@ #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/algorithm/string/predicate.hpp> #include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/preprocessor/stringize.hpp> @@ -5703,12 +5704,18 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass try { - std::stringstream iss; - iss << cache_data; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(cache_data)}; if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; + if (!loaded) + { + binary_archive<false> ar{epee::strspan<std::uint8_t>(cache_data)}; + ar.enable_varint_bug_backward_compatibility(); + if (::serialization::serialize(ar, *this)) + if (::serialization::check_stream_state(ar)) + loaded = true; + } } catch(...) { } @@ -6028,7 +6035,7 @@ uint64_t wallet2::balance(uint32_t index_major, bool strict) const { uint64_t amount = 0; if(m_light_wallet) - return m_light_wallet_unlocked_balance; + return m_light_wallet_balance; for (const auto& i : balance_per_subaddress(index_major, strict)) amount += i.second; return amount; @@ -6042,7 +6049,7 @@ uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t * if (time_to_unlock) *time_to_unlock = 0; if(m_light_wallet) - return m_light_wallet_balance; + return m_light_wallet_unlocked_balance; for (const auto& i : unlocked_balance_per_subaddress(index_major, strict)) { amount += i.second.first; @@ -6778,8 +6785,7 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi catch(const std::exception &e) { LOG_PRINT_L0("Failed to decrypt unsigned tx: " << e.what()); return false; } try { - std::istringstream iss(s); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(s)}; if (!::serialization::serialize(ar, exported_txs)) { LOG_PRINT_L0("Failed to parse data from unsigned tx"); @@ -7093,8 +7099,7 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too catch (const std::exception &e) { LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); return false; } try { - std::istringstream iss(s); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(s)}; if (!::serialization::serialize(ar, signed_txs)) { LOG_PRINT_L0("Failed to deserialize signed transaction"); @@ -7229,8 +7234,7 @@ bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx bool loaded = false; try { - std::istringstream iss(multisig_tx_st); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(multisig_tx_st)}; if (::serialization::serialize(ar, exported_txs)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -8753,7 +8757,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent // throw if total amount overflows uint64_t for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); @@ -8912,7 +8916,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry // throw if total amount overflows uint64_t for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); @@ -9885,14 +9889,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp needed_money = 0; for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, 0, m_nettype); } // throw if attempting a transaction with no money - THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_amount); std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false); @@ -11508,6 +11512,9 @@ void wallet2::check_tx_key_helper(const cryptonote::transaction &tx, const crypt void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) { + uint32_t rpc_version; + THROW_WALLET_EXCEPTION_IF(!check_connection(&rpc_version), error::wallet_internal_error, "Failed to connect to daemon: " + get_daemon_address()); + COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); @@ -11553,10 +11560,17 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de confirmations = 0; if (!in_pool) { - std::string err; - uint64_t bc_height = get_daemon_blockchain_height(err); - if (err.empty()) - confirmations = bc_height - res.txs.front().block_height; + if (rpc_version < MAKE_CORE_RPC_VERSION(3, 7)) + { + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (err.empty() && bc_height > res.txs.front().block_height) + confirmations = bc_height - res.txs.front().block_height; + } + else + { + confirmations = res.txs.front().confirmations; + } } } @@ -12019,8 +12033,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; try { - std::istringstream iss(sig_decoded); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(sig_decoded)}; if (::serialization::serialize_noeof(ar, proofs)) if (::serialization::serialize_noeof(ar, subaddr_spendkeys)) if (::serialization::check_stream_state(ar)) @@ -12220,7 +12233,7 @@ uint64_t wallet2::get_approximate_blockchain_height() const // Calculated blockchain height uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; // testnet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_testnet_rolled_back_blocks = 303967; + static const uint64_t approximate_testnet_rolled_back_blocks = 342100; if (m_nettype == TESTNET && approx_blockchain_height > approximate_testnet_rolled_back_blocks) approx_blockchain_height -= approximate_testnet_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); @@ -12531,7 +12544,7 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename, bool all) const { PERF_TIMER(export_key_images); - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const uint32_t offset = ski.first; @@ -12558,7 +12571,7 @@ bool wallet2::export_key_images(const std::string &filename, bool all) const } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const +std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const { PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; @@ -13055,7 +13068,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const +std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const { PERF_TIMER(export_outputs); std::vector<tools::wallet2::transfer_details> outs; @@ -13095,7 +13108,7 @@ std::string wallet2::export_outputs_to_str(bool all) const return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) +size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs) { PERF_TIMER(import_outputs); @@ -13201,12 +13214,10 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) try { std::string body(data, headerlen); - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs; try { - std::stringstream iss; - iss << body; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(body)}; if (::serialization::serialize(ar, outputs)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -13458,8 +13469,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) bool loaded = false; try { - std::istringstream iss(body); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(body)}; if (::serialization::serialize(ar, i)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -14181,15 +14191,15 @@ void wallet2::hash_m_transfer(const transfer_details & transfer, crypto::hash &h KECCAK_CTX state; keccak_init(&state); keccak_update(&state, (const uint8_t *) transfer.m_txid.data, sizeof(transfer.m_txid.data)); - keccak_update(&state, (const uint8_t *) transfer.m_internal_output_index, sizeof(transfer.m_internal_output_index)); - keccak_update(&state, (const uint8_t *) transfer.m_global_output_index, sizeof(transfer.m_global_output_index)); - keccak_update(&state, (const uint8_t *) transfer.m_amount, sizeof(transfer.m_amount)); + keccak_update(&state, (const uint8_t *) &transfer.m_internal_output_index, sizeof(transfer.m_internal_output_index)); + keccak_update(&state, (const uint8_t *) &transfer.m_global_output_index, sizeof(transfer.m_global_output_index)); + keccak_update(&state, (const uint8_t *) &transfer.m_amount, sizeof(transfer.m_amount)); keccak_finish(&state, (uint8_t *) hash.data); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const +uint64_t wallet2::hash_m_transfers(boost::optional<uint64_t> transfer_height, crypto::hash &hash) const { - CHECK_AND_ASSERT_THROW_MES(transfer_height > (int64_t)m_transfers.size(), "Hash height is greater than number of transfers"); + CHECK_AND_ASSERT_THROW_MES(!transfer_height || *transfer_height <= m_transfers.size(), "Hash height is greater than number of transfers"); KECCAK_CTX state; crypto::hash tmp_hash{}; @@ -14197,12 +14207,12 @@ uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) keccak_init(&state); for(const transfer_details & transfer : m_transfers){ - if (transfer_height >= 0 && current_height >= (uint64_t)transfer_height){ + if (transfer_height && current_height >= *transfer_height){ break; } hash_m_transfer(transfer, tmp_hash); - keccak_update(&state, (const uint8_t *) transfer.m_block_height, sizeof(transfer.m_block_height)); + keccak_update(&state, (const uint8_t *) &transfer.m_block_height, sizeof(transfer.m_block_height)); keccak_update(&state, (const uint8_t *) tmp_hash.data, sizeof(tmp_hash.data)); current_height += 1; } @@ -14214,23 +14224,28 @@ uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash) { // Compute hash of m_transfers, if differs there had to be BC reorg. - crypto::hash new_transfers_hash{}; - hash_m_transfers((int64_t) transfer_height, new_transfers_hash); + if (transfer_height <= m_transfers.size()) { + crypto::hash new_transfers_hash{}; + hash_m_transfers(transfer_height, new_transfers_hash); - if (new_transfers_hash != hash) - { - // Soft-Reset to avoid inconsistency in case of BC reorg. - clear_soft(false); // keep_key_images works only with soft reset. - THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, "Transfers changed during rescan, soft or hard rescan is needed"); - } + if (new_transfers_hash == hash) { + // Restore key images in m_transfers from m_key_images + for(auto it = m_key_images.begin(); it != m_key_images.end(); it++) + { + THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), + error::wallet_internal_error, + "Key images cache contains illegal transfer offset"); + m_transfers[it->second].m_key_image = it->first; + m_transfers[it->second].m_key_image_known = true; + } - // Restore key images in m_transfers from m_key_images - for(auto it = m_key_images.begin(); it != m_key_images.end(); it++) - { - THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), error::wallet_internal_error, "Key images cache contains illegal transfer offset"); - m_transfers[it->second].m_key_image = it->first; - m_transfers[it->second].m_key_image_known = true; + return; + } } + + // Soft-Reset to avoid inconsistency in case of BC reorg. + clear_soft(false); // keep_key_images works only with soft reset. + THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, "Transfers changed during rescan, soft or hard rescan is needed"); } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_bytes_sent() const diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index e5a5136a4..facf9878d 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -327,7 +327,7 @@ private: uint64_t m_block_height; cryptonote::transaction_prefix m_tx; crypto::hash m_txid; - size_t m_internal_output_index; + uint64_t m_internal_output_index; uint64_t m_global_output_index; bool m_spent; bool m_frozen; @@ -338,7 +338,7 @@ private: bool m_rct; bool m_key_image_known; bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested - size_t m_pk_index; + uint64_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; std::vector<rct::key> m_multisig_k; @@ -1372,9 +1372,9 @@ private: bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; std::string export_outputs_to_str(bool all = false) const; - size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); + size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1382,7 +1382,7 @@ private: std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename, bool all = false) const; - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); @@ -1428,7 +1428,7 @@ private: bool get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &hashing_blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie); bool daemon_requires_payment(); bool make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credits, uint64_t &balance); - bool search_for_rpc_payment(uint64_t credits_target, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc = NULL, const std::function<void(const std::string&)> &errorfunc = NULL); + bool search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc = NULL, const std::function<void(const std::string&)> &errorfunc = NULL); template<typename T> void handle_payment_changes(const T &res, std::true_type) { if (res.status == CORE_RPC_STATUS_OK || res.status == CORE_RPC_STATUS_PAYMENT_REQUIRED) m_rpc_payment_state.credits = res.credits; @@ -1547,7 +1547,7 @@ private: bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height); void hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const; - uint64_t hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const; + uint64_t hash_m_transfers(boost::optional<uint64_t> transfer_height, crypto::hash &hash) const; void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); void enable_dns(bool enable) { m_use_dns = enable; } void set_offline(bool offline = true); diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 4a89ed81a..011780f43 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -83,6 +83,7 @@ namespace tools // tx_rejected // tx_sum_overflow // tx_too_big + // zero_amount // zero_destination // wallet_rpc_error * // daemon_busy @@ -750,10 +751,18 @@ namespace tools uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- + struct zero_amount: public transfer_error + { + explicit zero_amount(std::string&& loc) + : transfer_error(std::move(loc), "destination amount is zero") + { + } + }; + //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error { explicit zero_destination(std::string&& loc) - : transfer_error(std::move(loc), "destination amount is zero") + : transfer_error(std::move(loc), "transaction has no destination") { } }; diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 6527b1384..bf278f695 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -42,6 +42,7 @@ #include "cryptonote_basic/blobdatatype.h" #include "common/i18n.h" #include "common/util.h" +#include "common/threadpool.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2.rpc_payments" @@ -101,7 +102,7 @@ bool wallet2::make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credit return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::search_for_rpc_payment(uint64_t credits_target, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc, const std::function<void(const std::string&)> &errorfunc) +bool wallet2::search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc, const std::function<void(const std::string&)> &errorfunc) { bool need_payment = false; bool payment_required; @@ -139,49 +140,65 @@ bool wallet2::search_for_rpc_payment(uint64_t credits_target, const std::functio continue; } - crypto::hash hash; - const uint32_t local_nonce = nonce++; // wrapping's OK - *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(local_nonce); - const uint8_t major_version = hashing_blob[0]; - if (major_version >= RX_BLOCK_VERSION) - { - const int miners = 1; - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, miners, 0); - } - else + if(n_threads == 0) + n_threads = boost::thread::hardware_concurrency(); + + std::vector<crypto::hash> hash(n_threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter(tpool); + + const uint32_t local_nonce = nonce += n_threads; // wrapping's OK + for (size_t i = 0; i < n_threads; i++) { - int cn_variant = hashing_blob[0] >= 7 ? hashing_blob[0] - 6 : 0; - crypto::cn_slow_hash(hashing_blob.data(), hashing_blob.size(), hash, cn_variant, height); + tpool.submit(&waiter, [&, i] { + *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(local_nonce-i); + const uint8_t major_version = hashing_blob[0]; + if (major_version >= RX_BLOCK_VERSION) + { + const int miners = 1; + crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data, miners, 0); + } + else + { + int cn_variant = hashing_blob[0] >= 7 ? hashing_blob[0] - 6 : 0; + crypto::cn_slow_hash(hashing_blob.data(), hashing_blob.size(), hash[i], cn_variant, height); + } + }); } - ++n_hashes; - if (cryptonote::check_hash(hash, diff)) + waiter.wait(); + n_hashes += n_threads; + + for(size_t i=0; i < n_threads; i++) { - uint64_t credits, balance; - try + if (cryptonote::check_hash(hash[i], diff)) { - make_rpc_payment(local_nonce, cookie, credits, balance); - if (credits != credits_per_hash_found) + uint64_t credits, balance; + try { - MERROR("Found nonce, but daemon did not credit us with the expected amount"); + make_rpc_payment(local_nonce-i, cookie, credits, balance); + if (credits != credits_per_hash_found) + { + MERROR("Found nonce, but daemon did not credit us with the expected amount"); + if (errorfunc) + errorfunc("Found nonce, but daemon did not credit us with the expected amount"); + return false; + } + MDEBUG("Found nonce " << local_nonce-i << " at diff " << diff << ", gets us " << credits_per_hash_found << ", now " << balance << " credits"); + if (!foundfunc(credits)) + break; + } + catch (const tools::error::wallet_coded_rpc_error &e) + { + MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); if (errorfunc) - errorfunc("Found nonce, but daemon did not credit us with the expected amount"); - return false; + errorfunc("Found nonce, but daemon errored out with error " + std::to_string(e.code()) + ": " + e.status() + ", continuing"); + } + catch (const std::exception &e) + { + MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); + if (errorfunc) + errorfunc("Found nonce, but daemon errored out with: '" + std::string(e.what()) + "', continuing"); } - MDEBUG("Found nonce " << local_nonce << " at diff " << diff << ", gets us " << credits_per_hash_found << ", now " << balance << " credits"); - if (!foundfunc(credits)) - break; - } - catch (const tools::error::wallet_coded_rpc_error &e) - { - MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); - if (errorfunc) - errorfunc("Found nonce, but daemon errored out with error " + std::to_string(e.code()) + ": " + e.status() + ", continuing"); - } - catch (const std::exception &e) - { - MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); - if (errorfunc) - errorfunc("Found nonce, but daemon errored out with: '" + std::string(e.what()) + "', continuing"); } } } diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 327a189ca..b72817ba0 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -762,6 +762,90 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_freeze(const wallet_rpc::COMMAND_RPC_FREEZE::request& req, wallet_rpc::COMMAND_RPC_FREEZE::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + if (req.key_image.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Must specify key image to freeze"); + return false; + } + crypto::key_image ki; + if (!epee::string_tools::hex_to_pod(req.key_image, ki)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; + er.message = "failed to parse key image"; + return false; + } + m_wallet->freeze(ki); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_thaw(const wallet_rpc::COMMAND_RPC_THAW::request& req, wallet_rpc::COMMAND_RPC_THAW::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + if (req.key_image.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Must specify key image to thaw"); + return false; + } + crypto::key_image ki; + if (!epee::string_tools::hex_to_pod(req.key_image, ki)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; + er.message = "failed to parse key image"; + return false; + } + m_wallet->thaw(ki); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_frozen(const wallet_rpc::COMMAND_RPC_FROZEN::request& req, wallet_rpc::COMMAND_RPC_FROZEN::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + if (req.key_image.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Must specify key image to check if frozen"); + return false; + } + crypto::key_image ki; + if (!epee::string_tools::hex_to_pod(req.key_image, ki)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; + er.message = "failed to parse key image"; + return false; + } + res.frozen = m_wallet->frozen(ki); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er) { crypto::hash8 integrated_payment_id = crypto::null_hash8; @@ -822,7 +906,7 @@ namespace tools if (at_least_one_destination && dsts.empty()) { er.code = WALLET_RPC_ERROR_CODE_ZERO_DESTINATION; - er.message = "No destinations for this transfer"; + er.message = "Transaction has no destination"; return false; } @@ -877,10 +961,10 @@ namespace tools return amount; } //------------------------------------------------------------------------------------------------------------------------------ - template<typename Ts, typename Tu> + template<typename Ts, typename Tu, typename Tk> bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, - Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er) + Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er) { for (const auto & ptx : ptx_vector) { @@ -895,6 +979,17 @@ namespace tools fill(amount, total_amount(ptx)); fill(fee, ptx.fee); fill(weight, cryptonote::get_transaction_weight(ptx.tx)); + + // add spent key images + tools::wallet_rpc::key_image_list key_image_list; + bool all_are_txin_to_key = std::all_of(ptx.tx.vin.begin(), ptx.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool + { + CHECKED_GET_SPECIFIC_VARIANT(s_e, const cryptonote::txin_to_key, in, false); + key_image_list.key_images.push_back(epee::string_tools::pod_to_hex(in.k_image)); + return true; + }); + THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, ptx.tx); + fill(spent_key_images, key_image_list); } if (m_wallet->multisig()) @@ -981,7 +1076,7 @@ namespace tools } return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); + res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er); } catch (const std::exception& e) { @@ -1027,7 +1122,7 @@ namespace tools } return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); + res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) { @@ -1389,7 +1484,7 @@ namespace tools std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); + res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) { @@ -1447,7 +1542,7 @@ namespace tools std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra, req.account_index, subaddr_indices); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); + res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) { @@ -1522,7 +1617,7 @@ namespace tools } return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); + res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er); } catch (const std::exception& e) { @@ -1555,8 +1650,7 @@ namespace tools try { - std::istringstream iss(blob); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(blob)}; if (::serialization::serialize(ar, ptx)) loaded = true; } @@ -1887,6 +1981,7 @@ namespace tools rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); rpc_transfers.subaddr_index = {td.m_subaddr_index.major, td.m_subaddr_index.minor}; rpc_transfers.key_image = td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; + rpc_transfers.pubkey = epee::string_tools::pod_to_hex(td.get_public_key()); rpc_transfers.block_height = td.m_block_height; rpc_transfers.frozen = td.m_frozen; rpc_transfers.unlocked = m_wallet->is_transfer_unlocked(td); @@ -2701,7 +2796,7 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); res.offset = ski.first; res.signed_key_images.resize(ski.second.size()); for (size_t n = 0; n < ski.second.size(); ++n) @@ -3013,6 +3108,41 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_scan_tx(const wallet_rpc::COMMAND_RPC_SCAN_TX::request& req, wallet_rpc::COMMAND_RPC_SCAN_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + + std::vector<crypto::hash> txids; + std::list<std::string>::const_iterator i = req.txids.begin(); + while (i != req.txids.end()) + { + cryptonote::blobdata txid_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob) || txid_blob.size() != sizeof(crypto::hash)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID; + er.message = "TX ID has invalid format"; + return false; + } + + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data()); + txids.push_back(txid); + } + + try { + m_wallet->scan_tx(txids); + } catch (const std::exception &e) { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -3118,17 +3248,7 @@ namespace tools } std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); { - std::vector<std::string> languages; - crypto::ElectrumWords::get_language_list(languages, false); - std::vector<std::string>::iterator it; - - it = std::find(languages.begin(), languages.end(), req.language); - if (it == languages.end()) - { - crypto::ElectrumWords::get_language_list(languages, true); - it = std::find(languages.begin(), languages.end(), req.language); - } - if (it == languages.end()) + if (!crypto::ElectrumWords::is_valid_language(req.language)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Unknown language: " + req.language; @@ -3333,6 +3453,11 @@ namespace tools er.code = WALLET_RPC_ERROR_CODE_DAEMON_IS_BUSY; er.message = e.what(); } + catch (const tools::error::zero_amount& e) + { + er.code = WALLET_RPC_ERROR_CODE_ZERO_AMOUNT; + er.message = e.what(); + } catch (const tools::error::zero_destination& e) { er.code = WALLET_RPC_ERROR_CODE_ZERO_DESTINATION; @@ -3546,6 +3671,17 @@ namespace tools return false; } + if (!req.language.empty()) + { + if (!crypto::ElectrumWords::is_valid_language(req.language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "The specified seed language is invalid."; + return false; + } + wal->set_seed_language(req.language); + } + // set blockheight if given try { @@ -3689,12 +3825,7 @@ namespace tools er.message = "Wallet was using the old seed language. You need to specify a new seed language."; return false; } - std::vector<std::string> language_list; - std::vector<std::string> language_list_en; - crypto::ElectrumWords::get_language_list(language_list); - crypto::ElectrumWords::get_language_list(language_list_en, true); - if (std::find(language_list.begin(), language_list.end(), req.language) == language_list.end() && - std::find(language_list_en.begin(), language_list_en.end(), req.language) == language_list_en.end()) + if (!crypto::ElectrumWords::is_valid_language(req.language)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Wallet was using the old seed language, and the specified new seed language is invalid."; diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 6e39eca1e..7169c9136 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -84,6 +84,9 @@ namespace tools MAP_JON_RPC_WE("set_account_tag_description", on_set_account_tag_description, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION) MAP_JON_RPC_WE("get_height", on_getheight, wallet_rpc::COMMAND_RPC_GET_HEIGHT) MAP_JON_RPC_WE("getheight", on_getheight, wallet_rpc::COMMAND_RPC_GET_HEIGHT) + MAP_JON_RPC_WE("freeze", on_freeze, wallet_rpc::COMMAND_RPC_FREEZE) + MAP_JON_RPC_WE("thaw", on_thaw, wallet_rpc::COMMAND_RPC_THAW) + MAP_JON_RPC_WE("frozen", on_frozen, wallet_rpc::COMMAND_RPC_FROZEN) MAP_JON_RPC_WE("transfer", on_transfer, wallet_rpc::COMMAND_RPC_TRANSFER) MAP_JON_RPC_WE("transfer_split", on_transfer_split, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT) MAP_JON_RPC_WE("sign_transfer", on_sign_transfer, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER) @@ -131,6 +134,7 @@ namespace tools MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("auto_refresh", on_auto_refresh, wallet_rpc::COMMAND_RPC_AUTO_REFRESH) + MAP_JON_RPC_WE("scan_tx", on_scan_tx, wallet_rpc::COMMAND_RPC_SCAN_TX) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) @@ -173,6 +177,9 @@ namespace tools bool on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_getheight(const wallet_rpc::COMMAND_RPC_GET_HEIGHT::request& req, wallet_rpc::COMMAND_RPC_GET_HEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_freeze(const wallet_rpc::COMMAND_RPC_FREEZE::request& req, wallet_rpc::COMMAND_RPC_FREEZE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_thaw(const wallet_rpc::COMMAND_RPC_THAW::request& req, wallet_rpc::COMMAND_RPC_THAW::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_frozen(const wallet_rpc::COMMAND_RPC_FROZEN::request& req, wallet_rpc::COMMAND_RPC_FROZEN::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); @@ -218,6 +225,7 @@ namespace tools bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_scan_tx(const wallet_rpc::COMMAND_RPC_SCAN_TX::request& req, wallet_rpc::COMMAND_RPC_SCAN_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); @@ -255,10 +263,10 @@ namespace tools bool not_open(epee::json_rpc::error& er); void handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code); - template<typename Ts, typename Tu> + template<typename Ts, typename Tu, typename Tk> bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, - Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er); + Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er); bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 81f83fb18..6640441ed 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 20 +#define WALLET_RPC_VERSION_MINOR 22 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -456,6 +456,78 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; + struct COMMAND_RPC_FREEZE + { + struct request_t + { + std::string key_image; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_image) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_THAW + { + struct request_t + { + std::string key_image; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_image) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_FROZEN + { + struct request_t + { + std::string key_image; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_image) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + bool frozen; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(frozen) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct key_image_list + { + std::list<std::string> key_images; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_images) + END_KV_SERIALIZE_MAP() + }; + struct COMMAND_RPC_TRANSFER { struct request_t @@ -499,6 +571,7 @@ namespace wallet_rpc std::string tx_metadata; std::string multisig_txset; std::string unsigned_txset; + key_image_list spent_key_images; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -510,6 +583,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -567,6 +641,7 @@ namespace wallet_rpc std::list<std::string> tx_metadata_list; std::string multisig_txset; std::string unsigned_txset; + std::list<key_image_list> spent_key_images_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -578,6 +653,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images_list) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -742,6 +818,7 @@ namespace wallet_rpc std::list<std::string> tx_metadata_list; std::string multisig_txset; std::string unsigned_txset; + std::list<key_image_list> spent_key_images_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -753,6 +830,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images_list) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -816,6 +894,7 @@ namespace wallet_rpc std::list<std::string> tx_metadata_list; std::string multisig_txset; std::string unsigned_txset; + std::list<key_image_list> spent_key_images_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -827,6 +906,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images_list) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -875,6 +955,7 @@ namespace wallet_rpc std::string tx_metadata; std::string multisig_txset; std::string unsigned_txset; + key_image_list spent_key_images; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -886,6 +967,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1010,6 +1092,7 @@ namespace wallet_rpc std::string tx_hash; cryptonote::subaddress_index subaddr_index; std::string key_image; + std::string pubkey; // owned output public key found uint64_t block_height; bool frozen; bool unlocked; @@ -1021,6 +1104,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) + KV_SERIALIZE(pubkey); KV_SERIALIZE(block_height) KV_SERIALIZE(frozen) KV_SERIALIZE(unlocked) @@ -2028,6 +2112,26 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_SCAN_TX + { + struct request_t + { + std::list<std::string> txids; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(txids) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_START_MINING { struct request_t @@ -2192,6 +2296,7 @@ namespace wallet_rpc std::string viewkey; std::string password; bool autosave_current; + std::string language; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_OPT(restore_height, (uint64_t)0) @@ -2201,6 +2306,7 @@ namespace wallet_rpc KV_SERIALIZE(viewkey) KV_SERIALIZE(password) KV_SERIALIZE_OPT(autosave_current, true) + KV_SERIALIZE(language) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index f7c5bb0e1..b991029a9 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -76,4 +76,5 @@ #define WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC -43 #define WALLET_RPC_ERROR_CODE_INVALID_LOG_LEVEL -44 #define WALLET_RPC_ERROR_CODE_ATTRIBUTE_NOT_FOUND -45 +#define WALLET_RPC_ERROR_CODE_ZERO_AMOUNT -46 #define WALLET_RPC_ERROR_CODE_INVALID_SIGNATURE_TYPE -47 diff --git a/tests/README.md b/tests/README.md index f6dd25ead..908482c99 100644 --- a/tests/README.md +++ b/tests/README.md @@ -50,7 +50,12 @@ To run the same tests on a release build, replace `debug` with `release`. # Functional tests [TODO] -Functional tests are located under the `tests/functional` directory. +Functional tests are located under the `tests/functional_tests` directory. + +Building all the tests requires installing the following dependencies: +```bash +pip install requests psutil monotonic +``` First, run a regtest daemon in the offline mode and with a fixed difficulty: ```bash @@ -65,6 +70,25 @@ velvet lymph giddy number token physics poetry unquoted nibs useful sabotage lim Open the wallet file with `monero-wallet-rpc` with RPC port 18083. Finally, start tests by invoking ./blockchain.py or ./speed.py +## Parameters + +Configuration of individual tests. + +### Mining test + +The following environment variables may be set to control the mining test: + +- `MINING_NO_MEASUREMENT` - set to anything to use large enough and fixed mining timeouts (use case: very slow PCs and no intention to change the mining code) +- `MINING_SILENT` - set to anything to disable mining logging + +For example, to customize the run of the functional tests, you may run the following commands from the build directory: + +```bash +export MINING_NO_MEASUREMENT=1 +ctest -V -R functional_tests_rpc +unset MINING_NO_MEASUREMENT +``` + # Fuzz tests Fuzz tests are written using American Fuzzy Lop (AFL), and located under the `tests/fuzz` directory. diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index a5fd35028..3f9f01a11 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -648,11 +648,9 @@ public: bvc.m_verifivation_failed = true; cryptonote::block blk; - std::stringstream ss; - ss << sr_block.data; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(sr_block.data)}; ::serialization::serialize(ba, blk); - if (!ss.good()) + if (!ba.good()) { blk = cryptonote::block(); } @@ -671,11 +669,9 @@ public: bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); cryptonote::transaction tx; - std::stringstream ss; - ss << sr_tx.data; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(sr_tx.data)}; ::serialization::serialize(ba, tx); - if (!ss.good()) + if (!ba.good()) { tx = cryptonote::transaction(); } @@ -1084,7 +1080,7 @@ inline bool do_replay_file(const std::string& filename) } #define QUOTEME(x) #x -#define DEFINE_TESTS_ERROR_CONTEXT(text) const char* perr_context = text; +#define DEFINE_TESTS_ERROR_CONTEXT(text) const char* perr_context = text; (void) perr_context; #define CHECK_TEST_CONDITION(cond) CHECK_AND_ASSERT_MES(cond, false, "[" << perr_context << "] failed: \"" << QUOTEME(cond) << "\"") #define CHECK_EQ(v1, v2) CHECK_AND_ASSERT_MES(v1 == v2, false, "[" << perr_context << "] failed: \"" << QUOTEME(v1) << " == " << QUOTEME(v2) << "\", " << v1 << " != " << v2) #define CHECK_NOT_EQ(v1, v2) CHECK_AND_ASSERT_MES(!(v1 == v2), false, "[" << perr_context << "] failed: \"" << QUOTEME(v1) << " != " << QUOTEME(v2) << "\", " << v1 << " == " << v2) diff --git a/tests/functional_tests/CMakeLists.txt b/tests/functional_tests/CMakeLists.txt index 462fd4b77..337097a34 100644 --- a/tests/functional_tests/CMakeLists.txt +++ b/tests/functional_tests/CMakeLists.txt @@ -64,16 +64,18 @@ target_link_libraries(make_test_signature ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) -execute_process(COMMAND ${PYTHON_EXECUTABLE} "-c" "import requests; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) +monero_add_minimal_executable(cpu_power_test cpu_power_test.cpp) +find_program(PYTHON3_FOUND python3 REQUIRED) + +execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) if (REQUESTS_OUTPUT STREQUAL "OK") add_test( NAME functional_tests_rpc - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/functional_tests_rpc.py" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" all) + COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/functional_tests_rpc.py" "${PYTHON3_FOUND}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" all) + add_test( + NAME check_missing_rpc_methods + COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}") else() - message(WARNING "functional_tests_rpc skipped, needs the 'requests' python module") - set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc) + message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil' and 'monotonic' python modules") + set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc check_missing_rpc_methods) endif() - -add_test( - NAME check_missing_rpc_methods - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}") diff --git a/tests/functional_tests/cpu_power_test.cpp b/tests/functional_tests/cpu_power_test.cpp new file mode 100644 index 000000000..54c4bfc9f --- /dev/null +++ b/tests/functional_tests/cpu_power_test.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2014-2021, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +const char * descr = +"This program prints the time (ms) needed to calculate a mathematical challenge. It's purpose \n\ +is to be able to use the result to compare the CPU power available on the machine it's being executed \n\ +and under the given circumstances, which can differ for the same machine. For example: \n\ +The printed value will be different when using 2 of 2 cores, when there an another process \n\ +running on one of the cores core, and when no other intense process running. \n\ + \n\ +The program expects a one argument: a numerical value of the threads to start the calculations on. \n\ + \n\ +Prints: \n\ +Time to calculate a mathematical challenge in MILLISECONDS. \n\ + \n\ +Returns: \n\ +0 on success, \n\ +1 on a missing argument, \n\ +2 on an incorrect format of the argument.\n"; + +#include <iostream> +#include <future> +#include <vector> +#include <sstream> +#include <chrono> + +using namespace std; + +/** +Uses Leibniz Formula for Pi. +https://en.wikipedia.org/wiki/Leibniz_formula_for_%CF%80 +*/ +static double calcPi(const size_t max_iter) +{ + const double n = 4; + double pi = 0; + double d = 1; + for (size_t i = 1; i < max_iter; ++i) + { + const double a = 2.0 * (i % 2) - 1.0; + pi += a * n / d; + d += 2; + } + return pi; +} + +int main(int argc, const char ** argv) +{ + const size_t max_iter = 1e9; + if (argc < 2) + { + cout << "Please pass the number of threads to run.\n"; + + cout << '\n' << descr << '\n'; + return 1; + } + + // Convert argument to an integer. + int numThreads = 1; + const char * numThreadsStr = argv[1]; + std::istringstream iss(numThreadsStr); + if (! (iss >> numThreads) ) + { + cout << "Incorrect format of number of threads = '" << numThreadsStr << "'\n"; + return 2; + } + // Run the calculation in parallel. + std::vector<std::future<double>> futures; + for(int i = 0; i < numThreads; ++i) + { + futures.push_back(std::async(calcPi, max_iter)); + } + + // Start measuring the time. + const std::chrono::steady_clock::time_point tbegin = std::chrono::steady_clock::now(); + for(auto & e : futures) + { + e.get(); + } + // Stop measuring the time. + const std::chrono::steady_clock::time_point tend = std::chrono::steady_clock::now(); + + // Print the measured duration. + cout << std::chrono::duration_cast<std::chrono::milliseconds>(tend - tbegin).count() << std::endl; + return 0; +} + diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 79e04b8a6..450552cf8 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -44,6 +44,7 @@ N_MONERODS = 4 N_WALLETS = 5 WALLET_DIRECTORY = builddir + "/functional-tests-directory" +FUNCTIONAL_TESTS_DIRECTORY = builddir + "/tests/functional_tests" DIFFICULTY = 10 monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", str(DIFFICULTY), "--no-igd", "--p2p-bind-port", "monerod_p2p_port", "--rpc-bind-port", "monerod_rpc_port", "--zmq-rpc-bind-port", "monerod_zmq_port", "--non-interactive", "--disable-dns-checkpoints", "--check-updates", "disabled", "--rpc-ssl", "disabled", "--data-dir", "monerod_data_dir", "--log-level", "1"] @@ -71,14 +72,14 @@ for i in range(N_MONERODS): command_lines.append([str(18180+i) if x == "monerod_rpc_port" else str(18280+i) if x == "monerod_p2p_port" else str(18380+i) if x == "monerod_zmq_port" else builddir + "/functional-tests-directory/monerod" + str(i) if x == "monerod_data_dir" else x for x in monerod_base]) if i < len(monerod_extra): command_lines[-1] += monerod_extra[i] - outputs.append(open(builddir + '/tests/functional_tests/monerod' + str(i) + '.log', 'a+')) + outputs.append(open(FUNCTIONAL_TESTS_DIRECTORY + '/monerod' + str(i) + '.log', 'a+')) ports.append(18180+i) for i in range(N_WALLETS): command_lines.append([str(18090+i) if x == "wallet_port" else x for x in wallet_base]) if i < len(wallet_extra): command_lines[-1] += wallet_extra[i] - outputs.append(open(builddir + '/tests/functional_tests/wallet' + str(i) + '.log', 'a+')) + outputs.append(open(FUNCTIONAL_TESTS_DIRECTORY + '/wallet' + str(i) + '.log', 'a+')) ports.append(18090+i) print('Starting servers...') @@ -89,9 +90,11 @@ try: PYTHONPATH += srcdir + '/../../utils/python-rpc' os.environ['PYTHONPATH'] = PYTHONPATH os.environ['WALLET_DIRECTORY'] = WALLET_DIRECTORY + os.environ['FUNCTIONAL_TESTS_DIRECTORY'] = FUNCTIONAL_TESTS_DIRECTORY + os.environ['SOURCE_DIRECTORY'] = srcdir os.environ['PYTHONIOENCODING'] = 'utf-8' os.environ['DIFFICULTY'] = str(DIFFICULTY) - os.environ['MAKE_TEST_SIGNATURE'] = builddir + '/tests/functional_tests/make_test_signature' + os.environ['MAKE_TEST_SIGNATURE'] = FUNCTIONAL_TESTS_DIRECTORY + '/make_test_signature' os.environ['SEEDHASH_EPOCH_BLOCKS'] = "8" os.environ['SEEDHASH_EPOCH_LAG'] = "4" diff --git a/tests/functional_tests/make_test_signature.cc b/tests/functional_tests/make_test_signature.cc index 07cc3eea6..cb1723847 100644 --- a/tests/functional_tests/make_test_signature.cc +++ b/tests/functional_tests/make_test_signature.cc @@ -28,6 +28,7 @@ #include <stdio.h> #include "misc_language.h" +#include "misc_log_ex.h" #include "string_tools.h" #include "rpc/rpc_payment_signature.h" diff --git a/tests/functional_tests/mining.py b/tests/functional_tests/mining.py index 50bfd32a2..7ecbdeed5 100755 --- a/tests/functional_tests/mining.py +++ b/tests/functional_tests/mining.py @@ -31,6 +31,10 @@ from __future__ import print_function import time import os +import math +import monotonic +import util_resources +import multiprocessing """Test daemon mining RPC calls @@ -38,6 +42,10 @@ Test the following RPCs: - start_mining - stop_mining - mining_status + +Control the behavior with these environment variables: + MINING_NO_MEASUREMENT - set to anything to use large enough and fixed mining timeouts + MINING_SILENT - set to anything to disable mining logging """ from framework.daemon import Daemon @@ -71,12 +79,23 @@ class MiningTest(): def mine(self, via_daemon): print("Test mining via " + ("daemon" if via_daemon else "wallet")) + cores_init = multiprocessing.cpu_count() # RX init uses all cores + cores_mine = 1 # Mining uses a parametric number of cores + is_mining_measurent = 'MINING_NO_MEASUREMENT' not in os.environ + + if is_mining_measurent: # A dynamic calculation of the CPU power requested + time_pi_single_cpu = self.measure_cpu_power_get_time(cores_mine) + time_pi_all_cores = self.measure_cpu_power_get_time(cores_init) + # This is the last measurement, since it takes very little time and can be placed timewise-closer to the mining itself. + available_ram = self.get_available_ram() # So far no ideas how to use this var, other than printing it + + start = monotonic.monotonic() daemon = Daemon() wallet = Wallet() # check info/height/balance before generating blocks res_info = daemon.get_info() - prev_height = res_info.height + initial_height = res_info.height res_getbalance = wallet.get_balance() prev_balance = res_getbalance.balance @@ -85,29 +104,75 @@ class MiningTest(): if via_daemon: res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1) else: - res = wallet.start_mining(threads_count = 1) + res = wallet.start_mining(threads_count = cores_mine) res_status = daemon.mining_status() assert res_status.active == True - assert res_status.threads_count == 1 + assert res_status.threads_count == cores_mine assert res_status.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' assert res_status.is_background_mining_enabled == False assert res_status.block_reward >= 600000000000 # wait till we mined a few of them - target_height = prev_height + 5 - height = prev_height - timeout = 240 # randomx is slow to init + target_height = initial_height + 5 + height = initial_height + + if not is_mining_measurent: + timeout_init = 600 + timeout_mine = 300 + else: + """ + Randomx init has high variance on CI machines due to noisy neighbors, + taking up resources in parallel (including by our own jobs). + + Mining is organized in the following scheme: + 1) first loop's pass: RandomX init and mining + 2) every next pass: only mining + Pass 1) takes much more time than pass 2) + Pass 1) uses all cores, pass 2) just one (currently) + For the above reasons both passes need separate timeouts and adjustments. + After the first pass, the timeout is being reset to a lower value. + """ + + def calc_timeout(seconds_constant, time_pi, cores): + """ + The time it took to calculate pi under certain conditions + is proportional to the time it will take to calculate the real job. + + The number of cores used decreases the time almost linearly. + """ + timeout = float(seconds_constant) * time_pi / float(cores) + return timeout + + timeout_base_init = 60 # RX init needs more time + timeout_base_mine = 20 + timeout_init = calc_timeout(timeout_base_init, time_pi_all_cores, cores_init) + timeout_mine = calc_timeout(timeout_base_mine, time_pi_single_cpu, cores_mine) + + msg_timeout_src = "adjusted for the currently available CPU power" if is_mining_measurent else "selected to have the default value" + msg = "Timeout for {} {}, is {:.1f} s" + self.print_mining_info(msg.format("init, ", msg_timeout_src, timeout_init)) + self.print_mining_info(msg.format("mining,", msg_timeout_src, timeout_mine)) + timeout = timeout_init + rx_inited = False # Gets initialized in the first pass of the below loop while height < target_height: seen_height = height - for _ in range(timeout): + for _ in range(int(math.ceil(timeout))): time.sleep(1) + seconds_passed = monotonic.monotonic() - start height = daemon.get_info().height if height > seen_height: break else: - assert False, 'Failed to mine successor to block %d (initial block = %d)' % (seen_height, prev_height) - timeout = 10 + assert False, 'Failed to mine successor to block %d (initial block = %d) after %d s. RX initialized = %r' % (seen_height, initial_height, round(seconds_passed), rx_inited) + if not rx_inited: + rx_inited = True + timeout = timeout_mine # Resetting the timeout after first mined block and RX init + self.print_time_taken(start, "RX init + mining 1st block") + else: + self.print_time_taken(start, "mining iteration") + + self.print_time_taken(start, "mining total") if via_daemon: res = daemon.stop_mining() @@ -123,7 +188,7 @@ class MiningTest(): wallet.refresh() res_getbalance = wallet.get_balance() balance = res_getbalance.balance - assert balance >= prev_balance + (new_height - prev_height) * 600000000000 + assert balance >= prev_balance + (new_height - initial_height) * 600000000000 if via_daemon: res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1, do_background_mining = True) @@ -144,6 +209,23 @@ class MiningTest(): res_status = daemon.mining_status() assert res_status.active == False + def measure_cpu_power_get_time(self, cores): + self.print_mining_info("Measuring the currently available CPU power...") + build_dir_funcional_tests = os.environ['FUNCTIONAL_TESTS_DIRECTORY'] + time_pi = util_resources.get_time_pi_seconds(cores, build_dir_funcional_tests) + self.print_mining_info("Time taken to calculate Pi on {} core(s) was {:.2f} s.".format(cores, time_pi)) + return time_pi + + def get_available_ram(self): + available_ram = util_resources.available_ram_gb() + threshold_ram = 3 + self.print_mining_info("Available RAM = " + str(round(available_ram, 1)) + " GB") + if available_ram < threshold_ram: + print("Warning! Available RAM =", round(available_ram, 1), + "GB is less than the reasonable threshold =", threshold_ram, + ". The RX init might exceed the calculated timeout.") + return available_ram + def submitblock(self): print("Test submitblock") @@ -171,6 +253,20 @@ class MiningTest(): assert res.height == height + i + 1 assert res.hash == block_hash + def is_mining_silent(self): + return 'MINING_SILENT' in os.environ + + def print_mining_info(self, msg): + if self.is_mining_silent(): + return + print(msg) + + def print_time_taken(self, start, msg_context): + if self.is_mining_silent(): + return + seconds_passed = monotonic.monotonic() - start + print("Time taken for", msg_context, "=", round(seconds_passed, 1), "s.") + def test_randomx(self): print("Test RandomX") diff --git a/tests/functional_tests/util_resources.py b/tests/functional_tests/util_resources.py new file mode 100755 index 000000000..0ea96c129 --- /dev/null +++ b/tests/functional_tests/util_resources.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" + Help determine how much CPU power is available at the given time + by running numerical calculations +""" + +from __future__ import print_function +import subprocess +import psutil + +def available_ram_gb(): + ram_bytes = psutil.virtual_memory().available + kilo = 1024.0 + ram_gb = ram_bytes / kilo**3 + return ram_gb + +def get_time_pi_seconds(cores, app_dir='.'): + app_path = '{}/cpu_power_test'.format(app_dir) + time_calc = subprocess.check_output([app_path, str(cores)]) + decoded = time_calc.decode('utf-8') + miliseconds = int(decoded) + + return miliseconds / 1000.0 diff --git a/tests/fuzz/bulletproof.cpp b/tests/fuzz/bulletproof.cpp index e0f183bc5..06416fe34 100644 --- a/tests/fuzz/bulletproof.cpp +++ b/tests/fuzz/bulletproof.cpp @@ -37,9 +37,7 @@ BEGIN_INIT_SIMPLE_FUZZER() END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::stringstream ss; - ss << std::string((const char*)buf, len); - binary_archive<false> ba(ss); + binary_archive<false> ba{{buf, len}}; rct::Bulletproof proof = AUTO_VAL_INIT(proof); ::serialization::serialize(ba, proof); END_SIMPLE_FUZZER() diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index 2698a36ba..69eb7b61f 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -50,11 +50,8 @@ BEGIN_INIT_SIMPLE_FUZZER() END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::string s((const char*)buf, len); - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; - std::stringstream iss; - iss << s; - binary_archive<false> ar(iss); + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs; + binary_archive<false> ar{{buf, len}}; ::serialization::serialize(ar, outputs); size_t n_outputs = wallet->import_outputs(outputs); std::cout << boost::lexical_cast<std::string>(n_outputs) << " outputs imported" << std::endl; diff --git a/tests/fuzz/cold-transaction.cpp b/tests/fuzz/cold-transaction.cpp index 135343704..bf5fba1f7 100644 --- a/tests/fuzz/cold-transaction.cpp +++ b/tests/fuzz/cold-transaction.cpp @@ -50,11 +50,8 @@ BEGIN_INIT_SIMPLE_FUZZER() END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::string s((const char*)buf, len); tools::wallet2::unsigned_tx_set exported_txs; - std::stringstream iss; - iss << s; - binary_archive<false> ar(iss); + binary_archive<false> ar{{buf, len}}; ::serialization::serialize(ar, exported_txs); std::vector<tools::wallet2::pending_tx> ptx; bool success = wallet->sign_tx(exported_txs, "/tmp/cold-transaction-test-signed", ptx); diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index 78b7b6863..209a75221 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -68,13 +68,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf.clone(); + buff_out.write(epee::to_span(m_invoke_out_buf)); return m_return_code; } diff --git a/tests/gtest/CMakeLists.txt b/tests/gtest/CMakeLists.txt index 621d0f042..50941b8d8 100644 --- a/tests/gtest/CMakeLists.txt +++ b/tests/gtest/CMakeLists.txt @@ -45,7 +45,7 @@ endif() # ${gtest_BINARY_DIR}. # Language "C" is required for find_package(Threads). project(gtest CXX C) -cmake_minimum_required(VERSION 2.6.2) +cmake_minimum_required(VERSION 3.5) if (COMMAND set_up_hermetic_build) set_up_hermetic_build() diff --git a/tests/gtest/cmake/internal_utils.cmake b/tests/gtest/cmake/internal_utils.cmake index ae45d7d7a..b33197989 100644 --- a/tests/gtest/cmake/internal_utils.cmake +++ b/tests/gtest/cmake/internal_utils.cmake @@ -96,7 +96,7 @@ macro(config_compiler_and_linker) set(cxx_no_rtti_flags "-fno-rtti -DGTEST_HAS_RTTI=0") set(cxx_strict_flags "-Wextra -Wno-unused-parameter -Wno-missing-field-initializers") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(cxx_base_flags "-Wall -Wshadow -fPIC") set(cxx_exception_flags "-fexceptions") set(cxx_no_exception_flags "-fno-exceptions") diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index baab07d31..59eef7bd1 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -67,7 +67,7 @@ namespace net_load_tests { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_connection_context& context) { //m_invoke_counter.inc(); //std::unique_lock<std::mutex> lock(m_mutex); diff --git a/tests/unit_tests/crypto.cpp b/tests/unit_tests/crypto.cpp index 7ac87ca7c..b550b35b0 100644 --- a/tests/unit_tests/crypto.cpp +++ b/tests/unit_tests/crypto.cpp @@ -33,6 +33,7 @@ #include <string> #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_basic/merge_mining.h" namespace { @@ -99,3 +100,215 @@ TEST(Crypto, verify_32) } } } + +TEST(Crypto, tree_branch) +{ + crypto::hash inputs[6]; + crypto::hash branch[8]; + crypto::hash root, root2; + size_t depth; + uint32_t path, path2; + + auto hasher = [](const crypto::hash &h0, const crypto::hash &h1) -> crypto::hash + { + char buffer[64]; + memcpy(buffer, &h0, 32); + memcpy(buffer + 32, &h1, 32); + crypto::hash res; + cn_fast_hash(buffer, 64, res); + return res; + }; + + for (int n = 0; n < 6; ++n) + { + memset(&inputs[n], 0, 32); + inputs[n].data[0] = n + 1; + } + + // empty + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 0, crypto::null_hash.data, (char(*)[32])branch, &depth, &path)); + + // one, matching + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 1, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_EQ(depth, 0); + ASSERT_EQ(path, 0); + ASSERT_TRUE(crypto::tree_path(1, 0, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 1, root.data); + ASSERT_EQ(root, inputs[0]); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // one, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 1, inputs[1].data, (char(*)[32])branch, &depth, &path)); + + // two, index 0 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 2, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_EQ(depth, 1); + ASSERT_EQ(path, 0); + ASSERT_TRUE(crypto::tree_path(2, 0, &path2)); + ASSERT_EQ(path, path2); + ASSERT_EQ(branch[0], inputs[1]); + crypto::tree_hash((const char(*)[32])inputs, 2, root.data); + ASSERT_EQ(root, hasher(inputs[0], inputs[1])); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // two, index 1 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 2, inputs[1].data, (char(*)[32])branch, &depth, &path)); + ASSERT_EQ(depth, 1); + ASSERT_EQ(path, 1); + ASSERT_TRUE(crypto::tree_path(2, 1, &path2)); + ASSERT_EQ(path, path2); + ASSERT_EQ(branch[0], inputs[0]); + crypto::tree_hash((const char(*)[32])inputs, 2, root.data); + ASSERT_EQ(root, hasher(inputs[0], inputs[1])); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // two, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 2, inputs[2].data, (char(*)[32])branch, &depth, &path)); + + // a b c 0 + // x y + // z + + // three, index 0 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 1); + ASSERT_LE(depth, 2); + ASSERT_TRUE(crypto::tree_path(3, 0, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 3, root.data); + ASSERT_EQ(root, hasher(inputs[0], hasher(inputs[1], inputs[2]))); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // three, index 1 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[1].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 1); + ASSERT_LE(depth, 2); + ASSERT_TRUE(crypto::tree_path(3, 1, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 3, root.data); + ASSERT_EQ(root, hasher(inputs[0], hasher(inputs[1], inputs[2]))); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // three, index 2 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[2].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 1); + ASSERT_LE(depth, 2); + ASSERT_TRUE(crypto::tree_path(3, 2, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 3, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::tree_branch_hash(inputs[2].data, (const char(*)[32])branch, depth, path, root2.data)); + ASSERT_EQ(root, root2); + + // three, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[3].data, (char(*)[32])branch, &depth, &path)); + + // a b c d e 0 0 0 + // x y + // z + // w + + // five, index 0 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 0, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // five, index 1 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[1].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 1, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // five, index 2 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[2].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 2, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // five, index 4 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[4].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 4, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth - 1, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth + 1, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path ^ 1)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path ^ 2)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path ^ 3)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])(branch + 1), depth, path)); + + // five, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 5, crypto::null_hash.data, (char(*)[32])branch, &depth, &path)); + + // depth encoding roundtrip + for (uint32_t n_chains = 1; n_chains <= 65; ++n_chains) + { + for (uint32_t nonce = 0; nonce < 1024; ++nonce) + { + const uint32_t depth = cryptonote::encode_mm_depth(n_chains, nonce); + uint32_t n_chains_2, nonce_2; + ASSERT_TRUE(cryptonote::decode_mm_depth(depth, n_chains_2, nonce_2)); + ASSERT_EQ(n_chains, n_chains_2); + ASSERT_EQ(nonce, nonce_2); + } + } +} diff --git a/tests/unit_tests/epee_boosted_tcp_server.cpp b/tests/unit_tests/epee_boosted_tcp_server.cpp index 06e076a3a..d10b2bb33 100644 --- a/tests/unit_tests/epee_boosted_tcp_server.cpp +++ b/tests/unit_tests/epee_boosted_tcp_server.cpp @@ -153,7 +153,7 @@ TEST(test_epee_connection, test_lifetime) delay(delay), on_connection_close_f(on_connection_close_f) {} - virtual int invoke(int, const epee::span<const uint8_t>, epee::byte_slice&, context_t&) override { epee::misc_utils::sleep_no_w(delay); return {}; } + virtual int invoke(int, const epee::span<const uint8_t>, epee::byte_stream&, context_t&) override { epee::misc_utils::sleep_no_w(delay); return {}; } virtual int notify(int, const epee::span<const uint8_t>, context_t&) override { return {}; } virtual void callback(context_t&) override {} virtual void on_connection_new(context_t&) override {} @@ -170,6 +170,7 @@ TEST(test_epee_connection, test_lifetime) using connection_ptr = boost::shared_ptr<connection_t>; using shared_state_t = typename connection_t::shared_state; using shared_state_ptr = std::shared_ptr<shared_state_t>; + using shared_states_t = std::vector<shared_state_ptr>; using tag_t = boost::uuids::uuid; using tags_t = std::vector<tag_t>; using io_context_t = boost::asio::io_service; @@ -281,7 +282,7 @@ TEST(test_epee_connection, test_lifetime) for (auto i = 0; i < N; ++i) { tag = create_connection(); ASSERT_TRUE(shared_state->get_connections_count() == 1); - success = shared_state->invoke_async(1, {}, tag, [](int, const epee::span<const uint8_t>, context_t&){}, TIMEOUT); + success = shared_state->invoke_async(1, epee::levin::message_writer{}, tag, [](int, const epee::span<const uint8_t>, context_t&){}, TIMEOUT); ASSERT_TRUE(success); while (shared_state->sock_count == 1) { success = shared_state->foreach_connection([&shared_state, &tag](context_t&){ @@ -319,7 +320,8 @@ TEST(test_epee_connection, test_lifetime) connection_ptr conn; { lock_guard_t guard(shared_conn->lock); - conn = std::move(shared_conn->conn.lock()); + conn = shared_conn->conn.lock(); + shared_conn->conn.reset(); } if (conn) conn->cancel(); @@ -344,6 +346,115 @@ TEST(test_epee_connection, test_lifetime) shared_state->del_out_connections(1); ASSERT_TRUE(shared_state->get_connections_count() == 0); } + + shared_states_t shared_states; + while (shared_states.size() < 2) { + shared_states.emplace_back(std::make_shared<shared_state_t>()); + shared_states.back()->set_handler(new command_handler_t(ZERO_DELAY, + [&shared_states]{ + for (auto &s: shared_states) { + auto success = s->foreach_connection([](context_t&){ + return true; + }); + ASSERT_TRUE(success); + } + } + ), + &command_handler_t::destroy + ); + } + workers_t workers; + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N * N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + io_context.post([conn]{ + conn->cancel(); + }); + conn.reset(); + s->del_out_connections(1); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N * N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + conn->cancel(); + while (conn.use_count() > 1); + s->foreach_connection([&io_context, &s, &endpoint, &conn](context_t& context){ + conn.reset(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + conn->cancel(); + while (conn.use_count() > 1); + conn.reset(); + return true; + }); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + context_t context; + conn->get_context(context); + auto tag = context.m_connection_id; + conn->cancel(); + while (conn.use_count() > 1); + s->for_connection(tag, [&io_context, &s, &endpoint, &conn](context_t& context){ + conn.reset(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + conn->cancel(); + while (conn.use_count() > 1); + conn.reset(); + return true; + }); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + context_t context; + conn->get_context(context); + auto tag = context.m_connection_id; + io_context.post([conn]{ + conn->cancel(); + }); + conn.reset(); + s->close(tag); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + }); for (auto& w: workers) { diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index a499fa608..d9de99b8b 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -59,13 +59,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf.clone(); + buff_out.write(epee::to_span(m_invoke_out_buf)); return m_return_code; } @@ -434,8 +434,11 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process const int expected_command = 4673261; const std::string in_data(256, 'e'); + epee::levin::message_writer message{}; + message.buffer.write(epee::to_span(in_data)); + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); + const epee::byte_slice notify = message.finalize_notify(expected_command); test_connection_ptr conn = create_connection(); @@ -468,11 +471,16 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process const int expected_command = 4673261; const int expected_fragmented_command = 46732; const std::string in_data(256, 'e'); - std::string in_fragmented_data(1024 * 4, 'c'); + + epee::levin::message_writer message{}; + message.buffer.write(epee::to_span(in_data)); + + epee::levin::message_writer in_fragmented_data; + in_fragmented_data.buffer.put_n('c', 1024 * 4); const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); - epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise, expected_fragmented_command, epee::strspan<std::uint8_t>(in_fragmented_data)); + const epee::byte_slice notify = message.finalize_notify(expected_command); + epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise.size(), expected_fragmented_command, std::move(in_fragmented_data)); EXPECT_EQ(5u, fragmented.size() / 1024); EXPECT_EQ(0u, fragmented.size() % 1024); @@ -497,11 +505,13 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process ASSERT_TRUE(conn->m_protocol_handler.handle_recv(next.data(), next.size())); } - in_fragmented_data.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + std::string compare_buffer(1024 * 4, 'c'); + compare_buffer.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + ASSERT_EQ(4u, m_commands_handler.notify_counter()); ASSERT_EQ(0u, m_commands_handler.invoke_counter()); ASSERT_EQ(expected_fragmented_command, m_commands_handler.last_command()); - ASSERT_EQ(in_fragmented_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(compare_buffer, m_commands_handler.last_in_buf()); ASSERT_EQ(0u, conn->send_counter()); ASSERT_TRUE(conn->last_send_data().empty()); diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 256f8c3c2..cbe3c61b1 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -1115,11 +1115,13 @@ TEST(ByteStream, ToByteSlice) epee::byte_stream stream; + stream.reserve(128*1024); stream.write(source); EXPECT_EQ(sizeof(source), stream.size()); + EXPECT_EQ(128*1024, stream.capacity()); EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()})); - const epee::byte_slice slice{std::move(stream)}; + const epee::byte_slice slice{std::move(stream), true}; EXPECT_EQ(0u, stream.size()); EXPECT_EQ(0u, stream.available()); EXPECT_EQ(0u, stream.capacity()); diff --git a/tests/unit_tests/get_xtype_from_string.cpp b/tests/unit_tests/get_xtype_from_string.cpp index 78bbe6657..17492ed19 100644 --- a/tests/unit_tests/get_xtype_from_string.cpp +++ b/tests/unit_tests/get_xtype_from_string.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include <string_tools.h> +#include <string_tools_lexical.h> using namespace epee::string_tools; diff --git a/tests/unit_tests/json_serialization.cpp b/tests/unit_tests/json_serialization.cpp index f76199e57..9fa589139 100644 --- a/tests/unit_tests/json_serialization.cpp +++ b/tests/unit_tests/json_serialization.cpp @@ -51,7 +51,7 @@ namespace test if (!cryptonote::find_tx_extra_field_by_type(extra_fields, key_field)) throw std::runtime_error{"invalid transaction"}; - for (auto const& input : boost::adaptors::index(source.vout)) + for (auto const input : boost::adaptors::index(source.vout)) { source_amount += input.value().amount; auto const& key = boost::get<cryptonote::txout_to_key>(input.value().target); diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index 53a7f7b67..30d6f8133 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -245,9 +245,9 @@ namespace return out; } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, cryptonote::levin::detail::p2p_context& context) override final + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, cryptonote::levin::detail::p2p_context& context) override final { - buff_out = nullptr; + buff_out.clear(); invoked_.push_back( {context.m_connection_id, command, std::string{reinterpret_cast<const char*>(in_buff.data()), in_buff.size()}} ); @@ -384,21 +384,50 @@ TEST(make_header, expect_return) EXPECT_EQ(0u, header1.m_flags); } -TEST(make_notify, empty_payload) +TEST(message_writer, invoke_with_empty_payload) { - const epee::byte_slice message = epee::levin::make_notify(443, nullptr); + const epee::byte_slice message = epee::levin::message_writer{}.finalize_invoke(443); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, true); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(message_writer, invoke_with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_invoke(443); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, true); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(message_writer, notify_with_empty_payload) +{ + const epee::byte_slice message = epee::levin::message_writer{}.finalize_notify(443); const epee::levin::bucket_head2 header = epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, false); ASSERT_EQ(sizeof(header), message.size()); EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); } -TEST(make_notify, with_payload) +TEST(message_writer, notify_with_payload) { std::string bytes(100, 'a'); std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); - const epee::byte_slice message = epee::levin::make_notify(443, epee::strspan<std::uint8_t>(bytes)); + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_notify(443); const epee::levin::bucket_head2 header = epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, false); @@ -407,6 +436,44 @@ TEST(make_notify, with_payload) EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); } +TEST(message_writer, response_with_empty_payload) +{ + const epee::byte_slice message = epee::levin::message_writer{}.finalize_response(443, 1); + epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_RESPONSE, false); + header.m_return_code = SWAP32LE(1); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(message_writer, response_with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_response(443, 6450); + epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_RESPONSE, false); + header.m_return_code = SWAP32LE(6450); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(message_writer, error) +{ + epee::levin::message_writer writer{}; + writer.buffer.clear(); + + EXPECT_THROW(writer.finalize_invoke(0), std::runtime_error); + EXPECT_THROW(writer.finalize_notify(0), std::runtime_error); + EXPECT_THROW(writer.finalize_response(0, 0), std::runtime_error); +} + TEST(make_noise, invalid) { EXPECT_TRUE(epee::levin::make_noise_notify(sizeof(epee::levin::bucket_head2) - 1).empty()); @@ -428,13 +495,13 @@ TEST(make_noise, valid) TEST(make_fragment, invalid) { - EXPECT_TRUE(epee::levin::make_fragmented_notify(nullptr, 0, nullptr).empty()); + EXPECT_TRUE(epee::levin::make_fragmented_notify(0, 0, epee::levin::message_writer{}).empty()); } TEST(make_fragment, single) { const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 11, nullptr); + const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise.size(), 11, epee::levin::message_writer{}); const epee::levin::bucket_head2 header = epee::levin::make_header(11, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_REQUEST, false); @@ -449,8 +516,13 @@ TEST(make_fragment, multiple) std::string bytes(1024 * 3 - 150, 'a'); std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + epee::levin::message_writer message; + message.buffer.write(epee::to_span(bytes)); + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 114, epee::strspan<std::uint8_t>(bytes)); + epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise.size(), 114, std::move(message)); + + EXPECT_EQ(1024 * 3, fragment.size()); epee::levin::bucket_head2 header = epee::levin::make_header(0, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_BEGIN, false); @@ -497,6 +569,7 @@ TEST(make_fragment, multiple) fragment.take_slice(bytes.size()); + EXPECT_EQ(18, fragment.size()); EXPECT_EQ(18, std::count(fragment.cbegin(), fragment.cend(), 0)); } @@ -664,8 +737,9 @@ TEST_F(levin_notify, stem_no_outs_without_padding) ASSERT_LT(0u, io_service_.poll()); EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff)); if (events_.has_stem_txes()) + { EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem)); - + } notifier.run_fluff(); ASSERT_LT(0u, io_service_.poll()); @@ -1034,7 +1108,9 @@ TEST_F(levin_notify, stem_no_outs_with_padding) ASSERT_LT(0u, io_service_.poll()); EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff)); if (events_.has_stem_txes()) + { EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem)); + } notifier.run_fluff(); ASSERT_LT(0u, io_service_.poll()); @@ -2161,20 +2237,31 @@ TEST_F(levin_notify, command_max_bytes) add_connection(true); - std::string bytes(4096, 'h'); + std::string payload(4096, 'h'); + epee::byte_slice bytes; + { + epee::levin::message_writer dest{}; + dest.buffer.write(epee::to_span(payload)); + bytes = dest.finalize_notify(ping_command); + } - EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1, get_connections().send(bytes.clone(), contexts_.front().get_id())); EXPECT_EQ(1u, contexts_.front().process_send_queue(true)); EXPECT_EQ(1u, receiver_.notified_size()); const received_message msg = receiver_.get_raw_notification(); EXPECT_EQ(ping_command, msg.command); EXPECT_EQ(contexts_.front().get_id(), msg.connection); - EXPECT_EQ(bytes, msg.payload); + EXPECT_EQ(payload, msg.payload); - bytes.push_back('e'); + { + payload.push_back('h'); + epee::levin::message_writer dest{}; + dest.buffer.write(epee::to_span(payload)); + bytes = dest.finalize_notify(ping_command); + } - EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1, get_connections().send(std::move(bytes), contexts_.front().get_id())); EXPECT_EQ(1u, contexts_.front().process_send_queue(false)); EXPECT_EQ(0u, receiver_.notified_size()); } diff --git a/tests/unit_tests/multiexp.cpp b/tests/unit_tests/multiexp.cpp index 722c568da..212aa0e40 100644 --- a/tests/unit_tests/multiexp.cpp +++ b/tests/unit_tests/multiexp.cpp @@ -260,7 +260,7 @@ TEST(multiexp, scalarmult_triple) rct::key res; ge_p3 Gp3; - ge_frombytes_vartime(&Gp3, rct::G.bytes); + ASSERT_EQ(ge_frombytes_vartime(&Gp3, rct::G.bytes), 0); static const rct::key scalars[] = { rct::Z, diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index af297d01a..2c80acda5 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -293,6 +293,18 @@ TEST(node_server, bind_same_p2p_port) boost::program_options::variables_map vm; boost::program_options::store(boost::program_options::parse_command_line(1, argv, desc_options), vm); + /* + Reason for choosing '127.0.0.2' as the IP: + + A TCP local socket address that has been bound is unavailable for some time after closing, unless the SO_REUSEADDR flag has been set. + That's why connections with automatically assigned source port 48080/58080 from previous test blocks the next to bind acceptor + so solution is to either set reuse_addr option for each socket in all tests + or use ip different from localhost for acceptors in order to not interfere with automatically assigned source endpoints + + Relevant part about REUSEADDR from man: + https://www.man7.org/linux/man-pages/man7/ip.7.html + */ + vm.find(nodetool::arg_p2p_bind_ip.name)->second = boost::program_options::variable_value(std::string("127.0.0.2"), false); vm.find(nodetool::arg_p2p_bind_port.name)->second = boost::program_options::variable_value(std::string(port), false); boost::program_options::notify(vm); @@ -310,5 +322,587 @@ TEST(node_server, bind_same_p2p_port) EXPECT_TRUE(init(new_node(), port_another)); } +TEST(cryptonote_protocol_handler, race_condition) +{ + struct contexts { + using basic = epee::net_utils::connection_context_base; + using cryptonote = cryptonote::cryptonote_connection_context; + using p2p = nodetool::p2p_connection_context_t<cryptonote>; + }; + using context_t = contexts::p2p; + using handler_t = epee::levin::async_protocol_handler<context_t>; + using connection_t = epee::net_utils::connection<handler_t>; + using connection_ptr = boost::shared_ptr<connection_t>; + using connections_t = std::vector<connection_ptr>; + using shared_state_t = typename connection_t::shared_state; + using shared_state_ptr = std::shared_ptr<shared_state_t>; + using io_context_t = boost::asio::io_service; + using event_t = epee::simple_event; + using ec_t = boost::system::error_code; + auto create_conn_pair = [](connection_ptr in, connection_ptr out) { + using endpoint_t = boost::asio::ip::tcp::endpoint; + using acceptor_t = boost::asio::ip::tcp::acceptor; + io_context_t io_context; + endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 5262); + acceptor_t acceptor(io_context); + ec_t ec; + acceptor.open(endpoint.protocol(), ec); + EXPECT_EQ(ec.value(), 0); + acceptor.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor.bind(endpoint, ec); + EXPECT_EQ(ec.value(), 0); + acceptor.listen(boost::asio::socket_base::max_connections, ec); + EXPECT_EQ(ec.value(), 0); + out->socket().open(endpoint.protocol(), ec); + EXPECT_EQ(ec.value(), 0); + acceptor.async_accept(in->socket(), [](const ec_t &ec){}); + out->socket().async_connect(endpoint, [](const ec_t &ec){}); + io_context.run(); + acceptor.close(ec); + EXPECT_EQ(ec.value(), 0); + EXPECT_TRUE(in->start(true, true)); + EXPECT_TRUE(out->start(false, true)); + return std::make_pair<>(std::move(in), std::move(out)); + }; + auto get_conn_tag = [](connection_t &conn){ + context_t context; + conn.get_context(context); + return context.m_connection_id; + }; + using work_t = boost::asio::io_service::work; + using work_ptr = std::shared_ptr<work_t>; + using workers_t = std::vector<std::thread>; + using commands_handler_t = epee::levin::levin_commands_handler<context_t>; + using p2p_endpoint_t = nodetool::i_p2p_endpoint<contexts::cryptonote>; + using core_t = cryptonote::core; + using core_ptr = std::unique_ptr<core_t>; + using core_protocol_t = cryptonote::t_cryptonote_protocol_handler<core_t>; + using core_protocol_ptr = std::shared_ptr<core_protocol_t>; + using block_t = cryptonote::block; + using diff_t = cryptonote::difficulty_type; + using reward_t = uint64_t; + using height_t = uint64_t; + struct span { + using blocks = epee::span<const block_t>; + }; + auto get_block_template = []( + core_t &core, + block_t &block, + diff_t &diff, + reward_t &reward + ){ + auto &storage = core.get_blockchain_storage(); + const auto height = storage.get_current_blockchain_height(); + const auto hardfork = storage.get_current_hard_fork_version(); + block.major_version = hardfork; + block.minor_version = storage.get_ideal_hard_fork_version(); + block.prev_id = storage.get_tail_id(); + auto &db = storage.get_db(); + block.timestamp = db.get_top_block_timestamp(); + block.nonce = 0xACAB; + block.miner_tx.vin.clear(); + block.miner_tx.vout.clear(); + block.miner_tx.extra.clear(); + block.miner_tx.version = hardfork >= 4 ? 2 : 1; + block.miner_tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; + block.miner_tx.vin.push_back(cryptonote::txin_gen{height}); + cryptonote::add_tx_pub_key_to_extra(block.miner_tx, {}); + cryptonote::get_block_reward( + db.get_block_weight(height - 1), + {}, + db.get_block_already_generated_coins(height - 1), + reward, + hardfork + ); + block.miner_tx.vout.push_back(cryptonote::tx_out{reward, cryptonote::txout_to_key{}}); + diff = storage.get_difficulty_for_next_block(); + }; + struct stat { + struct chain { + diff_t diff; + reward_t reward; + }; + }; + auto add_block = []( + core_t &core, + const block_t &block, + const stat::chain &stat + ){ + core.get_blockchain_storage().get_db().batch_start({}, {}); + core.get_blockchain_storage().get_db().add_block( + {block, cryptonote::block_to_blob(block)}, + cryptonote::get_transaction_weight(block.miner_tx), + core.get_blockchain_storage().get_next_long_term_block_weight( + cryptonote::get_transaction_weight(block.miner_tx) + ), + stat.diff, + stat.reward, + {} + ); + core.get_blockchain_storage().get_db().batch_stop(); + }; + struct messages { + struct core { + using sync = cryptonote::CORE_SYNC_DATA; + }; + using handshake = nodetool::COMMAND_HANDSHAKE_T<core::sync>; + }; + struct net_node_t: commands_handler_t, p2p_endpoint_t { + using span_t = epee::span<const uint8_t>; + using zone_t = epee::net_utils::zone; + using uuid_t = boost::uuids::uuid; + using relay_t = cryptonote::relay_method; + using blobs_t = std::vector<cryptonote::blobdata>; + using id_t = nodetool::peerid_type; + using callback_t = std::function<bool(contexts::cryptonote &, id_t, uint32_t)>; + using address_t = epee::net_utils::network_address; + using connections_t = std::vector<std::pair<zone_t, uuid_t>>; + struct bans { + using subnets = std::map<epee::net_utils::ipv4_network_subnet, time_t>; + using hosts = std::map<std::string, time_t>; + }; + shared_state_ptr shared_state; + core_protocol_ptr core_protocol; + virtual int invoke(int command, const span_t in, epee::byte_stream &out, context_t &context) override { + if (core_protocol) { + if (command == messages::handshake::ID) { + return epee::net_utils::buff_to_t_adapter<void, typename messages::handshake::request, typename messages::handshake::response>( + command, + in, + out, + [this](int command, typename messages::handshake::request &in, typename messages::handshake::response &out, context_t &context){ + core_protocol->process_payload_sync_data(in.payload_data, context, true); + core_protocol->get_payload_sync_data(out.payload_data); + return 1; + }, + context + ); + } + bool handled; + return core_protocol->handle_invoke_map(false, command, in, out, context, handled); + } + else + return {}; + } + virtual int notify(int command, const span_t in, context_t &context) override { + if (core_protocol) { + bool handled; + epee::byte_stream out; + return core_protocol->handle_invoke_map(true, command, in, out, context, handled); + } + else + return {}; + } + virtual void callback(context_t &context) override { + if (core_protocol) + core_protocol->on_callback(context); + } + virtual void on_connection_new(context_t&) override {} + virtual void on_connection_close(context_t &context) override { + if (core_protocol) + core_protocol->on_connection_close(context); + } + virtual ~net_node_t() override {} + virtual bool add_host_fail(const address_t&, unsigned int = {}) override { + return {}; + } + virtual bool block_host(address_t address, time_t = {}, bool = {}) override { + return {}; + } + virtual bool drop_connection(const contexts::basic& context) override { + if (shared_state) + return shared_state->close(context.m_connection_id); + else + return {}; + } + virtual bool for_connection(const uuid_t& uuid, callback_t f) override { + if (shared_state) + return shared_state->for_connection(uuid,[&f](context_t &context){ + return f(context, context.peer_id, context.support_flags); + }); + else + return {}; + } + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer in, const contexts::basic& context) override { + if (shared_state) + return shared_state->send(in.finalize_notify(command), context.m_connection_id); + else + return {}; + } + virtual bool relay_notify_to_list(int command, epee::levin::message_writer in, connections_t connections) override { + if (shared_state) { + for (auto &e: connections) + shared_state->send(in.finalize_notify(command), e.second); + } + return {}; + } + virtual bool unblock_host(const address_t&) override { + return {}; + } + virtual zone_t send_txs(blobs_t, const zone_t, const uuid_t&, relay_t) override { + return {}; + } + virtual bans::subnets get_blocked_subnets() override { + return {}; + } + virtual bans::hosts get_blocked_hosts() override { + return {}; + } + virtual uint64_t get_public_connections_count() override { + if (shared_state) + return shared_state->get_connections_count(); + else + return {}; + } + virtual void add_used_stripe_peer(const contexts::cryptonote&) override {} + virtual void clear_used_stripe_peers() override {} + virtual void remove_used_stripe_peer(const contexts::cryptonote&) override {} + virtual void for_each_connection(callback_t f) override { + if (shared_state) + shared_state->foreach_connection([&f](context_t &context){ + return f(context, context.peer_id, context.support_flags); + }); + } + virtual void request_callback(const contexts::basic &context) override { + if (shared_state) + shared_state->request_callback(context.m_connection_id); + } + }; + auto conduct_handshake = [get_conn_tag](net_node_t &net_node, connection_ptr conn){ + event_t handshaked; + net_node.shared_state->for_connection( + get_conn_tag(*conn), + [&handshaked, &net_node](context_t &context){ + typename messages::handshake::request msg; + net_node.core_protocol->get_payload_sync_data(msg.payload_data); + epee::net_utils::async_invoke_remote_command2<typename messages::handshake::response>( + context, + messages::handshake::ID, + msg, + *net_node.shared_state, + [&handshaked, &net_node](int code, const typename messages::handshake::response &msg, context_t &context){ + EXPECT_TRUE(code >= 0); + net_node.core_protocol->process_payload_sync_data(msg.payload_data, context, true); + handshaked.raise(); + }, + P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT + ); + return true; + } + ); + handshaked.wait(); + }; + using path_t = boost::filesystem::path; + auto create_dir = []{ + ec_t ec; + path_t path = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path("daemon-%%%%%%%%%%%%%%%%", ec); + if (ec) + return path_t{}; + auto success = boost::filesystem::create_directory(path, ec); + if (not ec && success) + return path; + return path_t{}; + }; + auto remove_tree = [](const path_t &path){ + ec_t ec; + boost::filesystem::remove_all(path, ec); + }; + using options_t = boost::program_options::variables_map; + struct daemon_t { + options_t options; + core_ptr core; + core_protocol_ptr core_protocol; + net_node_t net_node; + shared_state_ptr shared_state; + connections_t conn; + }; + struct daemons_t { + daemon_t main; + daemon_t alt; + }; + using options_description_t = boost::program_options::options_description; + + const auto dir = create_dir(); + ASSERT_TRUE(not dir.empty()); + + daemons_t daemon{ + { + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--data-dir", + (dir / "main").string(), + "--disable-dns-checkpoints", + "--check-updates=disabled", + "--fixed-difficulty=1", + "--block-sync-size=1", + "--db-sync-mode=fastest:async:50000", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }(), + {}, + {}, + {}, + {}, + {}, + }, + { + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--data-dir", + (dir / "alt").string(), + "--disable-dns-checkpoints", + "--check-updates=disabled", + "--fixed-difficulty=1", + "--block-sync-size=1", + "--db-sync-mode=fastest:async:50000", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }(), + {}, + {}, + {}, + {}, + {}, + }, + }; + + io_context_t io_context; + work_ptr work = std::make_shared<work_t>(io_context); + workers_t workers; + while (workers.size() < 4) { + workers.emplace_back([&io_context]{ + io_context.run(); + }); + } + + connection_t::set_rate_up_limit(std::numeric_limits<int64_t>::max()); + connection_t::set_rate_down_limit(std::numeric_limits<int64_t>::max()); + + { + daemon.main.core = core_ptr(new core_t(nullptr)); + daemon.main.core->init(daemon.main.options, nullptr, nullptr); + daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.main.core, &daemon.main.net_node, {} + )); + daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get()); + daemon.main.core_protocol->init(daemon.main.options); + daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared<shared_state_t>(); + daemon.main.shared_state->set_handler(&daemon.main.net_node); + daemon.alt.shared_state = std::make_shared<shared_state_t>(); + daemon.alt.shared_state->set_handler(&daemon.alt.net_node); + + struct { + event_t prepare; + event_t check; + event_t finish; + } events; + auto connections = create_conn_pair( + connection_ptr(new connection_t(io_context, daemon.main.shared_state, {}, {})), + connection_ptr(new connection_t(io_context, daemon.alt.shared_state, {}, {})) + ); + { + auto conn = connections.first; + auto shared_state = daemon.main.shared_state; + const auto tag = get_conn_tag(*conn); + conn->strand_.post([tag, conn, shared_state, &events]{ + shared_state->for_connection(tag, [](context_t &context){ + context.m_expect_height = -1; + context.m_expect_response = -1; + context.m_last_request_time = boost::date_time::min_date_time; + context.m_score = 0; + context.m_state = contexts::cryptonote::state_synchronizing; + return true; + }); + events.prepare.raise(); + events.check.wait(); + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_expect_height == -1); + EXPECT_TRUE(context.m_expect_response == -1); + EXPECT_TRUE(context.m_last_request_time == boost::date_time::min_date_time); + EXPECT_TRUE(context.m_score == 0); + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_synchronizing); + return true; + }); + events.finish.raise(); + }); + } + events.prepare.wait(); + daemon.main.core_protocol->on_idle(); + events.check.raise(); + events.finish.wait(); + + connections.first->strand_.post([connections]{ + connections.first->cancel(); + }); + connections.second->strand_.post([connections]{ + connections.second->cancel(); + }); + connections.first.reset(); + connections.second.reset(); + while (daemon.main.shared_state->sock_count); + while (daemon.alt.shared_state->sock_count); + daemon.main.core_protocol->deinit(); + daemon.main.core->stop(); + daemon.main.core->deinit(); + daemon.main.net_node.shared_state.reset(); + daemon.main.shared_state.reset(); + daemon.main.core_protocol.reset(); + daemon.main.core.reset(); + daemon.alt.shared_state.reset(); + } + + { + daemon.main.core = core_ptr(new core_t(nullptr)); + daemon.main.core->init(daemon.main.options, nullptr, nullptr); + daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.main.core, &daemon.main.net_node, {} + )); + daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get()); + daemon.main.core->set_checkpoints({}); + daemon.main.core_protocol->init(daemon.main.options); + daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared<shared_state_t>(); + daemon.main.shared_state->set_handler(&daemon.main.net_node); + daemon.alt.core = core_ptr(new core_t(nullptr)); + daemon.alt.core->init(daemon.alt.options, nullptr, nullptr); + daemon.alt.net_node.core_protocol = daemon.alt.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.alt.core, &daemon.alt.net_node, {} + )); + daemon.alt.core->set_cryptonote_protocol(daemon.alt.core_protocol.get()); + daemon.alt.core->set_checkpoints({}); + daemon.alt.core_protocol->init(daemon.alt.options); + daemon.alt.net_node.shared_state = daemon.alt.shared_state = std::make_shared<shared_state_t>(); + daemon.alt.shared_state->set_handler(&daemon.alt.net_node); + + struct { + io_context_t io_context; + work_ptr work; + workers_t workers; + } check; + check.work = std::make_shared<work_t>(check.io_context); + check.workers.emplace_back([&check]{ + check.io_context.run(); + }); + while (daemon.main.conn.size() < 1) { + daemon.main.conn.emplace_back(new connection_t(check.io_context, daemon.main.shared_state, {}, {})); + daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {})); + create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back()); + conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back()); + } + struct { + event_t prepare; + event_t sync; + event_t finish; + } events; + { + auto conn = daemon.main.conn.back(); + auto shared_state = daemon.main.shared_state; + const auto tag = get_conn_tag(*conn); + conn->strand_.post([tag, conn, shared_state, &events]{ + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal); + return true; + }); + events.prepare.raise(); + events.sync.wait(); + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal); + return true; + }); + events.finish.raise(); + }); + } + events.prepare.wait(); + daemon.main.core->get_blockchain_storage().add_block_notify( + [&events](height_t height, span::blocks blocks){ + if (height >= CRYPTONOTE_PRUNING_STRIPE_SIZE) + events.sync.raise(); + } + ); + { + stat::chain stat{ + daemon.alt.core->get_blockchain_storage().get_db().get_block_cumulative_difficulty( + daemon.alt.core->get_current_blockchain_height() - 1 + ), + daemon.alt.core->get_blockchain_storage().get_db().get_block_already_generated_coins( + daemon.alt.core->get_current_blockchain_height() - 1 + ), + }; + while (daemon.alt.core->get_current_blockchain_height() < CRYPTONOTE_PRUNING_STRIPE_SIZE + CRYPTONOTE_PRUNING_TIP_BLOCKS) { + block_t block; + diff_t diff; + reward_t reward; + get_block_template(*daemon.alt.core, block, diff, reward); + stat.diff += diff; + stat.reward = stat.reward < (MONEY_SUPPLY - stat.reward) ? stat.reward + reward : MONEY_SUPPLY; + add_block(*daemon.alt.core, block, stat); + if (daemon.main.core->get_current_blockchain_height() + 1 < CRYPTONOTE_PRUNING_STRIPE_SIZE) + add_block(*daemon.main.core, block, stat); + } + } + while (daemon.main.conn.size() < 2) { + daemon.main.conn.emplace_back(new connection_t(io_context, daemon.main.shared_state, {}, {})); + daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {})); + create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back()); + conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back()); + } + events.finish.wait(); + + for (;daemon.main.conn.size(); daemon.main.conn.pop_back()) { + auto conn = daemon.main.conn.back(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + } + for (;daemon.alt.conn.size(); daemon.alt.conn.pop_back()) { + auto conn = daemon.alt.conn.back(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + } + while (daemon.main.shared_state->sock_count); + while (daemon.alt.shared_state->sock_count); + daemon.main.core_protocol->deinit(); + daemon.main.core->stop(); + daemon.main.core->deinit(); + daemon.main.net_node.shared_state.reset(); + daemon.main.shared_state.reset(); + daemon.main.core_protocol.reset(); + daemon.main.core.reset(); + daemon.alt.core_protocol->deinit(); + daemon.alt.core->stop(); + daemon.alt.core->deinit(); + daemon.alt.net_node.shared_state.reset(); + daemon.alt.shared_state.reset(); + daemon.alt.core_protocol.reset(); + daemon.alt.core.reset(); + check.work.reset(); + for (auto& w: check.workers) { + w.join(); + } + } + + work.reset(); + for (auto& w: workers) { + w.join(); + } + remove_tree(dir); +} + namespace nodetool { template class node_server<cryptonote::t_cryptonote_protocol_handler<test_core>>; } namespace cryptonote { template class t_cryptonote_protocol_handler<test_core>; } diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 7b8a291d0..535752665 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -132,11 +132,10 @@ TEST(Serialization, BinaryArchiveInts) { ASSERT_EQ(8, oss.str().size()); ASSERT_EQ(string("\0\0\0\0\xff\0\0\0", 8), oss.str()); - istringstream iss(oss.str()); - binary_archive<false> iar(iss); + binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; iar.serialize_int(x1); - ASSERT_EQ(8, iss.tellg()); - ASSERT_TRUE(iss.good()); + ASSERT_EQ(8, iar.getpos()); + ASSERT_TRUE(iar.good()); ASSERT_EQ(x, x1); } @@ -151,10 +150,9 @@ TEST(Serialization, BinaryArchiveVarInts) { ASSERT_EQ(6, oss.str().size()); ASSERT_EQ(string("\x80\x80\x80\x80\xF0\x1F", 6), oss.str()); - istringstream iss(oss.str()); - binary_archive<false> iar(iss); + binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; iar.serialize_varint(x1); - ASSERT_TRUE(iss.good()); + ASSERT_TRUE(iar.good()); ASSERT_EQ(x, x1); } diff --git a/tests/unit_tests/test_tx_utils.cpp b/tests/unit_tests/test_tx_utils.cpp index 9425e2432..0d0dc8819 100644 --- a/tests/unit_tests/test_tx_utils.cpp +++ b/tests/unit_tests/test_tx_utils.cpp @@ -287,3 +287,73 @@ TEST(sort_tx_extra, invalid_suffix_partial) std::vector<uint8_t> expected(&expected_arr[0], &expected_arr[0] + sizeof(expected_arr)); ASSERT_EQ(sorted, expected); } + +TEST(remove_field_from_tx_extra, remove_first) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 1, 42}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(2, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[1].type()); + + tx_extra_fields.clear(); + ASSERT_TRUE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_pub_key))); + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(1, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[0].type()); +} + +TEST(remove_field_from_tx_extra, remove_last) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 1, 42}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(2, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[1].type()); + + tx_extra_fields.clear(); + ASSERT_TRUE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_nonce))); + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(1, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); +} + +TEST(remove_field_from_tx_extra, remove_middle) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 1, 42, 1, 30, 208, 98, 162, 133, 64, 85, 83, 112, + 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(3, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[1].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[2].type()); + + tx_extra_fields.clear(); + ASSERT_TRUE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_nonce))); + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(2, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); +} + +TEST(remove_field_from_tx_extra, invalid_varint) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 0x80, 0}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + ASSERT_FALSE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_nonce))); + ASSERT_EQ(sizeof(extra_arr), extra.size()); +} diff --git a/tests/unit_tests/tx_proof.cpp b/tests/unit_tests/tx_proof.cpp index c5d06bc68..0adb8713e 100644 --- a/tests/unit_tests/tx_proof.cpp +++ b/tests/unit_tests/tx_proof.cpp @@ -58,14 +58,14 @@ TEST(tx_proof, prove_verify_v2) // R_B = rB crypto::public_key R_B; ge_p3 B_p3; - ge_frombytes_vartime(&B_p3,&B); + ASSERT_EQ(ge_frombytes_vartime(&B_p3,&B), 0); ge_p2 R_B_p2; ge_scalarmult(&R_B_p2, &unwrap(r), &B_p3); ge_tobytes(&R_B, &R_B_p2); // R_G = rG crypto::public_key R_G; - ge_frombytes_vartime(&B_p3,&B); + ASSERT_EQ(ge_frombytes_vartime(&B_p3,&B), 0); ge_p3 R_G_p3; ge_scalarmult_base(&R_G_p3, &unwrap(r)); ge_p3_tobytes(&R_G, &R_G_p3); @@ -73,7 +73,7 @@ TEST(tx_proof, prove_verify_v2) // D = rA crypto::public_key D; ge_p3 A_p3; - ge_frombytes_vartime(&A_p3,&A); + ASSERT_EQ(ge_frombytes_vartime(&A_p3,&A), 0); ge_p2 D_p2; ge_scalarmult(&D_p2, &unwrap(r), &A_p3); ge_tobytes(&D, &D_p2); diff --git a/translations/CMakeLists.txt b/translations/CMakeLists.txt index 0081746f2..927d1733a 100644 --- a/translations/CMakeLists.txt +++ b/translations/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.5) project(translations) @@ -73,7 +73,7 @@ string(REPLACE ".ts" ".qm" qm_files "${ts_files}") add_custom_command(TARGET generate_translations_header POST_BUILD - COMMAND ./generate_translations_header ${qm_files} + COMMAND $<TARGET_FILE:generate_translations_header> ${qm_files} WORKING_DIRECTORY "${CMAKE_CURRENT_BIN_DIR}" COMMENT "Generating embedded translations header") diff --git a/utils/build_scripts/android32.Dockerfile b/utils/build_scripts/android32.Dockerfile index a2d0edbb3..c0931ce05 100644 --- a/utils/build_scripts/android32.Dockerfile +++ b/utils/build_scripts/android32.Dockerfile @@ -44,7 +44,7 @@ ARG BOOST_VERSION=1_68_0 ARG BOOST_VERSION_DOT=1.68.0 ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7 RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ + && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOT}/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && rm -f boost_${BOOST_VERSION}.tar.bz2 \ diff --git a/utils/build_scripts/android64.Dockerfile b/utils/build_scripts/android64.Dockerfile index eca2d4da1..3a62da464 100644 --- a/utils/build_scripts/android64.Dockerfile +++ b/utils/build_scripts/android64.Dockerfile @@ -44,7 +44,7 @@ ARG BOOST_VERSION=1_68_0 ARG BOOST_VERSION_DOT=1.68.0 ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7 RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ + && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOT}/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && rm -f boost_${BOOST_VERSION}.tar.bz2 \ diff --git a/utils/gpg_keys/anon.asc b/utils/gpg_keys/anon.asc new file mode 100644 index 000000000..83e1a0cd3 --- /dev/null +++ b/utils/gpg_keys/anon.asc @@ -0,0 +1,13 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEAAAAGBYJKwYBBAHaRw8BAQdAtk3N9Qcw9x3s5frY00j/jO10wfOCZ59mTLtN +tsYJ1620GGFub24gPGFub24gW2F0XSBub3doZXJlPoiQBBMWCAA4FiEEn2DdNuXc +KHIjsNT304V8F6p/losFAmAhnpUCGyMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AA +CgkQ04V8F6p/loumGwEAxjmzDa6ZK7zsps0sYbOvlh0tmxByUqZy1GvIaiOpdboA +/2guWA0tzK+fyQS2L2oXzFwh+oTk5bZ2KqnnfCr9a6kDuDgEYCG1oRIKKwYBBAGX +VQEFAQEHQCnTdYrNM/yw++BSJPgPAoImGhrKaqBoe5VBi9TUk+8aAwEIB4h4BBgW +CAAgFiEEn2DdNuXcKHIjsNT304V8F6p/losFAmAhtaECGwwACgkQ04V8F6p/lose +3AD9E3e4olBj7e6t5NtlMkkBzYKs16RNZRMJ5GMOw8lH8NUA/imPWARO2Y4ryswV +LAtQE1x0SCd4wEP8QEta4p2OYscI +=9Mvr +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/mj-xmr.asc b/utils/gpg_keys/mj-xmr.asc new file mode 100644 index 000000000..e02d3e511 --- /dev/null +++ b/utils/gpg_keys/mj-xmr.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGBxrvMBEADD7dDjSfrtiuQJEuMMMGYt3wGQ1iJdiLWe4THJDbtPDCLdBxh4 +fpYGMVkUYT8LJFkf41JD/hAIKMNQHdw7qmZcfN9F/9U+jInHRYBM1dbA/mHs8Jhi +YEGrD+v7c5fEuFXANW7z2uPjszw67tv3K5z8pFkt6K0pCgnxSKBu10WYkV0fvnxz +e3naXAnWQQ/qeGP2xT2F3S+bn5GvCTPw1pI1HAaFcYVD+XZpNdKQHnHMWlDrQ0qe +zymqrknjMmtD70Z7gEOogNxjd2QXUsQDq7XwxfHnhVZHHm98O95gh4Uw2Pve/TMI +f0ItA+MYp+hDqN7wmq3TjNSVI9L02m/RgpO7cxpEgQ9QFYJo3dHC8cP7x/uqH3vI +usYjOJtzq3UZWn4hnHRviEMYSDPtfFmO5CRElueVsRTiG0h7cyQYZ8HrEC9qDyR8 +vn0shqrUbcwsplyhwtpoyuJKcK/mFnmETUQ7ZF5NqgIlOaU7Du8WBt84YrDKRe5q +TKMTtDEf4PrejqhYvb7GNOK3D/LJ/dZTJIdETZ2rRCCftgKjTLJr9fRSrQnxw5NQ +8fnnK/O8JNpRvnDvR2HlmI8w/sgEDT99U4XgSi9BIUrM8HYkNykENXsk2BvEIGEN +txXwZKDJmxK787+y4WX2JfbhFGPNCXxuwrXUqNhKy4Qit140Z7WH6UIvSwARAQAB +tEdtai14bXIgKEJlY2F1c2UgV29ybGQgZG9taW5hdGlvbiBzaG91bGQgYmUgZnVu +LikgPG1qeG1yQHByb3Rvbm1haWwuY29tPokCVAQTAQoAPhYhBLMwkesQonK53VcC ++sEBv5QJNFHgBQJgca7zAhsDBQkJZgGABQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA +AAoJEMEBv5QJNFHgPykP/0YOJYU0ap8BpAtG9Dfv46UPy77XMw2ag2AJ0MYPrqy6 +omjha15qLHeXxCr3CJNYa4EEvX0hahSGn4k3VmcDHVYYpD4f/RZfu8fBpk0xQtsv +wF7zuAn0KSeg7maopdWfcPn02uTOm2crzTMMm5scXZ/YmMviUFPsgpyIkM5oCre6 +/RhIK953OUpgc/JKIlhpABA7KBTWUbc18BGHHdcPntVKrIxrIvzchVwI2P8KDQki +NS33Fc9EoOlDuzbTkQldZJrsEsMZWKfzoNNlZsLtroaS6SlqQNllYjSEJx7ccjEU ++zvMGLXhxqL7XTFO1HdvacWwPsSPR8MXrwUFcKrDDzC57rHNPrHncPY1JwEkN0Aj +V3VzTM+5QPvo6QQ+aNISxy9Hs470SpLDmmBl//Dn4qbFWSqAQ3qSD7S34MgW0FeC +4L1/uTsDokkx+ZIf7WQUl2p4vi8/puFw2y3VLs4WGaUYko1gjhoFIDs3gP7YIeE9 +Lp+6/+RxMkuSGTrHM9rcpeClp3jEN/I7n/epKpJOSgroN+/RdbOOnVECC7+kWkM0 +U9fPi+O3fRIlaR+Q8NWkokgkbtw6gYV3bKnW78AnF5INs3rrtOH1WtGSp7ghKP4p +7GE5cCIbUsB3dT6Ifo5tAi3faYFyhUZ2GsYNM0fsge10OwtPs+4YbSuurWUyIwuW +uQINBGBxrvMBEACxiDPPivT0Tv4DWtkGZ7tHRF0IT9nSZuL5F+qMX4D3hef3RfVY +4hxAQe2elvMU4PV/3neaRyiWSd4/Q8DnWrGQGvn3gxx8AHZTQwv/01Ae0InyNI9t +pzpBCMvOY73dba1JRklEMMyJPAUrdvmyRTEuZEIAxoVlaKKSAr4oeaENGLwWxB4o +NsAPxKNWw8drGQfJixMA+IeqeMzRjo6o5gJzrlaTb3+S78a1rD0pMmY7s6j3LqmF +kd0L/JlGrs+ILCxXjddg6FRL/qUtvOZX+jdogb1hKvPFCMMc1BfUdq9WbrCpoj54 +OhBnO+yCHi2faUi0YP8Fumu6RTCDN+MK0nEji+peaJApHf15ZVd9DAX9NzRl/iLb +sMjf4KBE2BI6FaUpdzkTbH6F713O2ZGWGEROBTjKucO3OkjGfHKf9rxdGRt37I/z +b59xRzSn91Vz1nEXCmDSO/iPAGZ3bQKh+zSGwjwJsZrBRCvS34AhGaEs8un6nd9M +FoMClpyjmm3W6WUZnELmcdlKhY2xbi5Yub21+mrjjjS6BuFpSYIC1xKJrCdpXtAm +i8x5mcKbY24uRuqsY9pQrDHVXK/Pwj6+k9rGwbe5P6VkbZOpJ2pWSL68EoxIo1zp +bv6moIJtxipydUOD2k76W0oGplexoxtW5f1WKcb3ykJRRrdPQxtPaPK14QARAQAB +iQI8BBgBCgAmFiEEszCR6xCicrndVwL6wQG/lAk0UeAFAmBxrvMCGwwFCQlmAYAA +CgkQwQG/lAk0UeClZhAAwQa+XO2GhCfZgrYOOSR5h2iib5k8OWiTsV68/DHOMsFJ +1WVDsqjlNEOhPPg0CbH4FMHJOndcsjJb+Rs6Y35YsgTmQl1SHlJXZinV1AJ+IaW/ +Esx+qOpoGUzCzCrjGpwSYHxGSQ52WpLvTaHbosH3x2XXTZ2znOdwTlFB/36Yco3c +ROWVLT/FhjugxTP0KddZR9Lu7ThK7txQhTQp5oBwWgpv0YAPaRPA+BpvrtzI7xSx +2AOMtr7jiGAu0yFL5WestRy37UmunxS8Rd8PGILcfFBIvfqNTMwU55SW1nS9XbRh +bUb/iil+Ly6lVsGAPW7iUgEUO7HtsPxeF+Bk/Hh+ZCHglQrGpzGcPmgbG3IiMF9H +nISbh6sFrkDXjyLNP1zl6wg0cTyk12OcYo0nX4jp1CY3Oztd3rVdcMT8Qot4JSHj +rDx+b3FTErVCnitEeALuBt/JSNH0VMcgOmFen9goJRVXcW/Dnsb+7QWXooj3aC++ +O6eghDN1YyJClQqLxvVThr7pdlMiWgWzHLmRkI2o84r14qOC15yTh7qr3Y5LOMPN +px9K2f9f2l6KJ0w7vgKwa6pYiVf1AAw2Ym98LAJn+6v9+nRx9mSq9Nw+M5eo9DXW +gclwPV1SSWUls8jjOmluI6/j8TE9JMBy5UKX1g6TOKJWW6wXcOUfzNVtG1oRNPA= +=egkb +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/health/README.md b/utils/health/README.md index 8fadd3908..05eb057a1 100644 --- a/utils/health/README.md +++ b/utils/health/README.md @@ -1,26 +1,32 @@ -#Intro +# Intro + This directory contains tools, which can be used for checking the health of the project, like build/run time analyzers, lints, etc. -#Usage +# Usage + Unless it's stated differently, these scripts should be called from a given source directory, where you want the checks to be performed, for instance: `og@ghetto:~/dev/monero$ utils/health/clang-build-time-analyzer-run.sh` -##ClangBuildAnalyzer +## ClangBuildAnalyzer + `utils/health/clang-build-time-analyzer-run.sh` The CBA helps in finding culprints of slow compilation. On the first run, the script will complain about the missing ClangBuildAnalyzer binary and will point you to another script, which is able to clone and build the required binary. -##clang-tidy +## clang-tidy + `utils/health/clang-tidy-run.sh` Performs Lint checks on the source code and stores the result in the build directory. More information on the [home page](https://clang.llvm.org/extra/clang-tidy/). -##include-what-you-use +## include-what-you-use + `utils/health/clang-include-what-you-use-run.sh` Analyses the header file hierarchy and delivers hints on how to reduce their complexity. More information on the [home page](https://include-what-you-use.org/). -##Valgrind checks +## Valgrind checks + `utils/health/valgrind-tests.sh` This script is able to run valgrind's callgrind, cachegrind and memcheck for a given set of executables. It expects ONE PARAMETER, which points to a file with paths to executables and their arguments, written line by line. For example: @@ -33,6 +39,7 @@ build/tests/unit_tests/unit_tests The `*.out` results can be interpreted with the `kcachegrind` tool. The memcheck output is just a readable text file with a summary at the end. -#Footer +# Footer + Responsible: mj-xmr diff --git a/utils/python-rpc/console.py b/utils/python-rpc/console.py index c94e07eab..4b3c18f5d 100755 --- a/utils/python-rpc/console.py +++ b/utils/python-rpc/console.py @@ -4,7 +4,16 @@ from __future__ import print_function import sys import subprocess import socket -import urlparse +try: + import urllib.parse + url_parser = urllib.parse.urlparse +except: + try: + import urlparse + url_parser = urlparse.urlparse + except: + print('urllib or urlparse is needed') + sys.exit(1) import framework.rpc import framework.daemon import framework.wallet @@ -21,7 +30,7 @@ for n in range(1, len(sys.argv)): try: port = int(sys.argv[n]) except: - t = urlparse.urlparse(sys.argv[n], allow_fragments = False) + t = url_parser(sys.argv[n], allow_fragments = False) scheme = t.scheme or scheme host = t.hostname or host port = t.port or port diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 435bc93f8..207565e6f 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -53,6 +53,19 @@ class Daemon(object): return self.rpc.send_json_rpc_request(getblocktemplate) get_block_template = getblocktemplate + def add_aux_pow(self, blocktemplate_blob, aux_pow, client = ""): + add_aux_pow = { + 'method': 'add_aux_pow', + 'params': { + 'blocktemplate_blob': blocktemplate_blob, + 'aux_pow' : aux_pow, + 'client' : client, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(add_aux_pow) + def send_raw_transaction(self, tx_as_hex, do_not_relay = False, do_sanity_checks = True, client = ""): send_raw_transaction = { 'client': client, diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index d97c24143..02084620c 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -1088,3 +1088,47 @@ class Wallet(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_version) + + def scan_tx(self, txids): + scan_tx = { + 'method': 'scan_tx', + 'jsonrpc': '2.0', + 'params' : { + 'txids': txids, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(scan_tx) + + def freeze(self, key_image): + freeze = { + 'method': 'freeze', + 'jsonrpc': '2.0', + 'params' : { + 'key_image': key_image, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(freeze) + + def thaw(self, key_image): + thaw = { + 'method': 'thaw', + 'jsonrpc': '2.0', + 'params' : { + 'key_image': key_image, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(thaw) + + def frozen(self, key_image): + frozen = { + 'method': 'frozen', + 'jsonrpc': '2.0', + 'params' : { + 'key_image': key_image, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(frozen) |