diff options
153 files changed, 2952 insertions, 802 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index c19c9dba7..54bc948e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,7 +187,7 @@ if(NOT MANUAL_SUBMODULES) if (upToDate) message(STATUS "Submodule '${relative_path}' is up-to-date") else() - message(FATAL_ERROR "Submodule '${relative_path}' is not up-to-date. Please update with\ngit submodule update --init --force ${relative_path}\nor run cmake with -DMANUAL_SUBMODULES=1") + message(FATAL_ERROR "Submodule '${relative_path}' is not up-to-date. Please update all submodules with\ngit submodule update --init --force\nor run cmake with -DMANUAL_SUBMODULES=1\n") endif() endfunction () @@ -246,6 +246,12 @@ enable_testing() option(BUILD_DOCUMENTATION "Build the Doxygen documentation." ON) option(BUILD_TESTS "Build tests." OFF) +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(DEFAULT_BUILD_DEBUG_UTILITIES ON) +else() + set(DEFAULT_BUILD_DEBUG_UTILITIES OFF) +endif() +option(BUILD_DEBUG_UTILITIES "Build debug utilities." DEFAULT_BUILD_DEBUG_UTILITIES) # Check whether we're on a 32-bit or 64-bit system if(CMAKE_SIZEOF_VOID_P EQUAL "8") @@ -672,8 +678,11 @@ else() add_cxx_flag_if_supported(-fstack-clash-protection CXX_SECURITY_FLAGS) endif() - add_c_flag_if_supported(-mmitigate-rop C_SECURITY_FLAGS) - add_cxx_flag_if_supported(-mmitigate-rop CXX_SECURITY_FLAGS) + # Removed in GCC 9.1 (or before ?), but still accepted, so spams the output + if (NOT (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1)) + add_c_flag_if_supported(-mmitigate-rop C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-mmitigate-rop CXX_SECURITY_FLAGS) + endif() # linker if (NOT WIN32) @@ -898,9 +907,9 @@ elseif(Boost_FOUND) set(BOOST_BEFORE_1_62 true) endif() if (BOOST_BEFORE_1_62) - message(FATAL_ERROR "Boost older than 1.62 is too old to link with OpenSSL 1.1 or newer. " + message(FATAL_ERROR "Boost ${Boost_VERSION} (older than 1.62) is too old to link with OpenSSL ${OPENSSL_VERSION} (1.1 or newer) found at ${OPENSSL_INCLUDE_DIR} and ${OPENSSL_LIBRARIES}. " "Update Boost or install OpenSSL 1.0 and set path to it when running cmake: " - "cmake -DOPENSSL_ROOT_DIR='/usr/include/openssl-1.0;/usr/lib/openssl-1.0'") + "cmake -DOPENSSL_ROOT_DIR='/usr/include/openssl-1.0'") endif() endif() @@ -995,7 +1004,16 @@ add_subdirectory(contrib) add_subdirectory(src) if(BUILD_TESTS) + message(STATUS "Building tests") add_subdirectory(tests) +else() + message(STATUS "Not building tests") +endif() + +if(BUILD_DEBUG_UTILITIES) + message(STATUS "Building debug utilities") +else() + message(STATUS "Not building debug utilities") endif() if(BUILD_DOCUMENTATION) diff --git a/README.i18n.md b/README.i18n.md index a3bd8070e..5df277624 100644 --- a/README.i18n.md +++ b/README.i18n.md @@ -15,23 +15,33 @@ You do not need anything from Qt in order to use the final translations. To update ts files after changing source code: - ./utils/translations/update-translations.sh +```bash +./utils/translations/update-translations.sh +``` To add a new language, eg Spanish (ISO code es): - cp translations/monero.ts translations/monero_es.ts +```bash +cp translations/monero.ts translations/monero_es.ts +``` To edit translations for Spanish: - linguist translations/monero_es.ts +```bash +linguist translations/monero_es.ts +``` To build translations after modifying them: - ./utils/translations/build-translations.sh +```bash +./utils/translations/build-translations.sh +``` To test a translation: - LANG=es ./build/release/bin/monero-wallet-cli +```bash +LANG=es ./build/release/bin/monero-wallet-cli +``` To add new translatable strings in the source code: @@ -39,6 +49,8 @@ Use the `tr(string)` function if possible. If the code is in a class, and this c If you're getting messages of the form: - Class 'cryptonote::simple_wallet' lacks Q_OBJECT macro +``` +Class 'cryptonote::simple_wallet' lacks Q_OBJECT macro +``` all is fine, we don't actually need that here. @@ -1,8 +1,28 @@ # Monero -Copyright (c) 2014-2018 The Monero Project. +Copyright (c) 2014-2019 The Monero Project. Portions Copyright (c) 2012-2013 The Cryptonote developers. +## Table of Contents + + - [Development resources](#development-resources) + - [Vulnerability response](#vulnerability-response) + - [Research](#research) + - [Announcements](#announcements) + - [Translations](#translations) + - [Build](#build) + - [IMPORTANT](#important) + - [Coverage](#coverage) + - [Introduction](#introduction) + - [About this project](#about-this-project) + - [Supporting the project](#supporting-the-project) + - [License](#license) + - [Contributing](#contributing) + - [Scheduled software upgrades](#scheduled-software-upgrades) + - [Release staging schedule and protocol](#release-staging-schedule-and-protocol) + - [Compiling Monero from source](#compiling-monero-from-source) + - [Dependencies](#dependencies) + ## Development resources - Web: [getmonero.org](https://getmonero.org) @@ -117,7 +137,7 @@ Monero uses a fixed-schedule software upgrade (hard fork) mechanism to implement Dates are provided in the format YYYY-MM-DD. -| Software upgrade block height | Date | Fork version | Minimum Monero version | Recommended Monero version | Details | +| Software upgrade block height | Date | Fork version | Minimum Monero version | Recommended Monero version | Details | | ------------------------------ | -----------| ----------------- | ---------------------- | -------------------------- | ---------------------------------------------------------------------------------- | | 1009827 | 2016-03-22 | v2 | v0.9.4 | v0.9.4 | Allow only >= ringsize 3, blocktime = 120 seconds, fee-free blocksize 60 kb | | 1141317 | 2016-09-21 | v3 | v0.9.4 | v0.10.0 | Splits coinbase into denominations | @@ -127,8 +147,8 @@ Dates are provided in the format YYYY-MM-DD. | 1546000 | 2018-04-06 | v7 | v0.12.0.0 | v0.12.3.0 | Cryptonight variant 1, ringsize >= 7, sorted inputs | 1685555 | 2018-10-18 | v8 | v0.13.0.0 | v0.13.0.4 | max transaction size at half the penalty free block size, bulletproofs enabled, cryptonight variant 2, fixed ringsize [11](https://youtu.be/KOO5S4vxi0o) | 1686275 | 2018-10-19 | v9 | v0.13.0.0 | v0.13.0.4 | bulletproofs required -| 1788000 | 2019-03-09 | v10 | v0.14.0.0 | v0.14.0.2 | New PoW based on Cryptonight-R, new block weight algorithm, slightly more efficient RingCT format -| 1788720 | 2019-03-10 | v11 | v0.14.0.0 | v0.14.0.2 | forbid old RingCT transaction format +| 1788000 | 2019-03-09 | v10 | v0.14.0.0 | v0.14.1.2 | New PoW based on Cryptonight-R, new block weight algorithm, slightly more efficient RingCT format +| 1788720 | 2019-03-10 | v11 | v0.14.0.0 | v0.14.1.2 | forbid old RingCT transaction format | XXXXXXX | 2019-10-XX | XX | XXXXXXXXX | XXXXXXXXX | X X's indicate that these details have not been determined as of commit date. @@ -206,9 +226,11 @@ invokes cmake commands as needed. * Install the dependencies * Change to the root of the source code directory, change to the most recent release branch, and build: - cd monero - git checkout release-v0.14 - make + ```bash + cd monero + git checkout release-v0.14 + make + ``` *Optional*: If your machine has several cores and enough memory, enable parallel build by running `make -j<number of threads>` instead of `make`. For @@ -232,23 +254,31 @@ invokes cmake commands as needed. * **Optional**: build and run the test suite to verify the binaries: - make release-test + ```bash + make release-test + ``` *NOTE*: `core_tests` test may take a few hours to complete. * **Optional**: to build binaries suitable for debugging: - make debug + ```bash + make debug + ``` * **Optional**: to build statically-linked binaries: - make release-static + ```bash + make release-static + ``` Dependencies need to be built with -fPIC. Static libraries usually aren't, so you may have to build them yourself with -fPIC. Refer to their documentation for how to build them. * **Optional**: build documentation in `doc/html` (omit `HAVE_DOT=YES` if `graphviz` is not installed): - HAVE_DOT=YES doxygen Doxyfile + ```bash + HAVE_DOT=YES doxygen Doxyfile + ``` #### On the Raspberry Pi @@ -259,24 +289,30 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( * Install the dependencies for Monero from the 'Debian' column in the table above. * Increase the system swap size: -``` - sudo /etc/init.d/dphys-swapfile stop - sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=2048 - sudo /etc/init.d/dphys-swapfile start -``` + + ```bash + sudo /etc/init.d/dphys-swapfile stop + sudo nano /etc/dphys-swapfile + CONF_SWAPSIZE=2048 + sudo /etc/init.d/dphys-swapfile start + ``` + * If using an external hard disk without an external power supply, ensure it gets enough power to avoid hardware issues when syncing, by adding the line "max_usb_current=1" to /boot/config.txt * Clone monero and checkout the most recent release version: -``` - git clone https://github.com/monero-project/monero.git - cd monero - git checkout tags/v0.14.1.0 -``` + + ```bash + git clone https://github.com/monero-project/monero.git + cd monero + git checkout tags/v0.14.1.2 + ``` + * Build: -``` - make release -``` + + ```bash + make release + ``` + * Wait 4-6 hours * The resulting executables can be found in `build/release/bin` @@ -293,28 +329,33 @@ If you are using the older Raspbian Jessie image, compiling Monero is a bit more * As before, `apt-get update && apt-get upgrade` to install all of the latest software, and increase the system swap size -``` - sudo /etc/init.d/dphys-swapfile stop - sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=2048 - sudo /etc/init.d/dphys-swapfile start -``` + ```bash + sudo /etc/init.d/dphys-swapfile stop + sudo nano /etc/dphys-swapfile + CONF_SWAPSIZE=2048 + sudo /etc/init.d/dphys-swapfile start + ``` + * Then, install the dependencies for Monero except `libunwind` and `libboost-all-dev` * Install the latest version of boost (this may first require invoking `apt-get remove --purge libboost*` to remove a previous version if you're not using a clean install): -``` - cd - wget https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 - tar xvfo boost_1_64_0.tar.bz2 - cd boost_1_64_0 - ./bootstrap.sh - sudo ./b2 -``` + + ```bash + cd + wget https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 + tar xvfo boost_1_64_0.tar.bz2 + cd boost_1_64_0 + ./bootstrap.sh + sudo ./b2 + ``` + * Wait ~8 hours -``` - sudo ./bjam cxxflags=-fPIC cflags=-fPIC -a install -``` + + ```bash + sudo ./bjam cxxflags=-fPIC cflags=-fPIC -a install + ``` + * Wait ~4 hours * From here, follow the [general Raspberry Pi instructions](#on-the-raspberry-pi) from the "Clone monero and checkout most recent release version" step. @@ -333,24 +374,32 @@ application. * Open the MSYS shell via the `MSYS2 Shell` shortcut * Update packages using pacman: - pacman -Syu + ```bash + pacman -Syu + ``` * Exit the MSYS shell using Alt+F4 * Edit the properties for the `MSYS2 Shell` shortcut changing "msys2_shell.bat" to "msys2_shell.cmd -mingw64" for 64-bit builds or "msys2_shell.cmd -mingw32" for 32-bit builds * Restart MSYS shell via modified shortcut and update packages again using pacman: - pacman -Syu + ```bash + pacman -Syu + ``` * Install dependencies: To build for 64-bit Windows: - pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi + ```bash + pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi + ``` To build for 32-bit Windows: - pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi + ```bash + pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi + ``` * Open the MingW shell via `MinGW-w64-Win64 Shell` shortcut on 64-bit Windows or `MinGW-w64-Win64 Shell` shortcut on 32-bit Windows. Note that if you are @@ -360,35 +409,49 @@ application. * To git clone, run: - git clone --recursive https://github.com/monero-project/monero.git + ```bash + git clone --recursive https://github.com/monero-project/monero.git + ``` **Building** * Change to the cloned directory, run: - cd monero + ```bash + cd monero + ``` -* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.14.1.0'. If you don't care about the version and just want binaries from master, skip this step: +* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.14.1.2'. If you don't care about the version and just want binaries from master, skip this step: - git checkout v0.14.1.0 + ```bash + git checkout v0.14.1.2 + ``` * If you are on a 64-bit system, run: - make release-static-win64 + ```bash + make release-static-win64 + ``` * If you are on a 32-bit system, run: - make release-static-win32 + ```bash + make release-static-win32 + ``` * The resulting executables can be found in `build/release/bin` * **Optional**: to build Windows binaries suitable for debugging on a 64-bit system, run: - make debug-static-win64 + ```bash + make debug-static-win64 + ``` * **Optional**: to build Windows binaries suitable for debugging on a 32-bit system, run: - make debug-static-win32 + ```bash + make debug-static-win32 + ``` * The resulting executables can be found in `build/debug/bin` @@ -428,7 +491,7 @@ We assume you are compiling with a non-root user and you have `doas` enabled. Note: do not use the boost package provided by OpenBSD, as we are installing boost to `/usr/local`. -``` +```bash # Create boost building directory mkdir ~/boost cd ~/boost @@ -464,7 +527,7 @@ Build the cppzmq bindings. We assume you are compiling with a non-root user and you have `doas` enabled. -``` +```bash # Create cppzmq building directory mkdir ~/cppzmq cd ~/cppzmq @@ -484,7 +547,10 @@ cmake .. doas make install ``` -Build monero: `env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local make release-static` +Build monero: +```bash +env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local make release-static +``` #### OpenBSD >= 6.4 @@ -507,23 +573,27 @@ Then you need to increase the data ulimit size to 2GB and try again: `ulimit -d The default Solaris linker can't be used, you have to install GNU ld, then run cmake manually with the path to your copy of GNU ld: - mkdir -p build/release - cd build/release - cmake -DCMAKE_LINKER=/path/to/ld -D CMAKE_BUILD_TYPE=Release ../.. - cd ../.. +```bash +mkdir -p build/release +cd build/release +cmake -DCMAKE_LINKER=/path/to/ld -D CMAKE_BUILD_TYPE=Release ../.. +cd ../.. +``` Then you can run make as usual. ### On Linux for Android (using docker): - # Build image (for ARM 32-bit) - docker build -f utils/build_scripts/android32.Dockerfile -t monero-android . - # Build image (for ARM 64-bit) - docker build -f utils/build_scripts/android64.Dockerfile -t monero-android . - # Create container - docker create -it --name monero-android monero-android bash - # Get binaries - docker cp monero-android:/src/build/release/bin . +```bash +# Build image (for ARM 32-bit) +docker build -f utils/build_scripts/android32.Dockerfile -t monero-android . +# Build image (for ARM 64-bit) +docker build -f utils/build_scripts/android64.Dockerfile -t monero-android . +# Create container +docker create -it --name monero-android monero-android bash +# Get binaries +docker cp monero-android:/src/build/release/bin . +``` ### Building portable statically linked binaries @@ -542,12 +612,18 @@ By default, in either dynamically or statically linked builds, binaries target t You can also cross-compile static binaries on Linux for Windows and macOS with the `depends` system. * ```make depends target=x86_64-linux-gnu``` for 64-bit linux binaries. -* ```make depends target=x86_64-w64-mingw32``` for 64-bit windows binaries. Requires: python3 g++-mingw-w64-x86-64 wine1.6 bc -* ```make depends target=x86_64-apple-darwin11``` for macOS binaries. Requires: cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev -* ```make depends target=i686-linux-gnu``` for 32-bit linux binaries. Requires: g++-multilib bc -* ```make depends target=i686-w64-mingw32``` for 32-bit windows binaries. Requires: python3 g++-mingw-w64-i686 -* ```make depends target=arm-linux-gnueabihf``` for armv7 binaries. Requires: g++-arm-linux-gnueabihf -* ```make depends target=aarch64-linux-gnu``` for armv8 binaries. Requires: g++-aarch64-linux-gnu +* ```make depends target=x86_64-w64-mingw32``` for 64-bit windows binaries. + * Requires: `python3 g++-mingw-w64-x86-64 wine1.6 bc` +* ```make depends target=x86_64-apple-darwin11``` for macOS binaries. + * Requires: `cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev` +* ```make depends target=i686-linux-gnu``` for 32-bit linux binaries. + * Requires: `g++-multilib bc` +* ```make depends target=i686-w64-mingw32``` for 32-bit windows binaries. + * Requires: `python3 g++-mingw-w64-i686` +* ```make depends target=arm-linux-gnueabihf``` for armv7 binaries. + * Requires: `g++-arm-linux-gnueabihf` +* ```make depends target=aarch64-linux-gnu``` for armv8 binaries. + * Requires: `g++-aarch64-linux-gnu` The required packages are the names for each toolchain on apt. Depending on your distro, they may have different names. @@ -563,7 +639,9 @@ Packages are available for * Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. - snap install monero --beta + ```bash + snap install monero --beta + ``` Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. @@ -573,25 +651,31 @@ Installing a snap is very quick. Snaps are secure. They are isolated with all of * Void Linux: - xbps-install -S monero + ```bash + xbps-install -S monero + ``` * GuixSD - guix package -i monero + ```bash + guix package -i monero + ``` * Docker - # Build using all available cores - docker build -t monero . - - # or build using a specific number of cores (reduce RAM requirement) - docker build --build-arg NPROC=1 -t monero . - - # either run in foreground - docker run -it -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero - - # or in background - docker run -it -d -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + ```bash + # Build using all available cores + docker build -t monero . + + # or build using a specific number of cores (reduce RAM requirement) + docker build --build-arg NPROC=1 -t monero . + + # either run in foreground + docker run -it -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + + # or in background + docker run -it -d -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + ``` * The build needs 3 GB space. * Wait one hour or more @@ -604,7 +688,9 @@ The build places the binary in `bin/` sub-directory within the build directory from which cmake was invoked (repository root by default). To run in foreground: - ./bin/monerod +```bash +./bin/monerod +``` To list all available options, run `./bin/monerod --help`. Options can be specified either on the command line or in a configuration file passed by the @@ -614,7 +700,9 @@ of the argument without the leading dashes, for example `log-level=1`. To run in background: - ./bin/monerod --log-file monerod.log --detach +```bash +./bin/monerod --log-file monerod.log --detach +``` To run as a systemd service, copy [monerod.service](utils/systemd/monerod.service) to `/etc/systemd/system/` and @@ -662,7 +750,9 @@ setting the following configuration parameters and environment variables: Example command line to start monerod through Tor: - DNS_PUBLIC=tcp torsocks monerod --p2p-bind-ip 127.0.0.1 --no-igd +```bash +DNS_PUBLIC=tcp torsocks monerod --p2p-bind-ip 127.0.0.1 --no-igd +``` ### Using Tor on Tails @@ -670,9 +760,11 @@ TAILS ships with a very restrictive set of firewall rules. Therefore, you need to add a rule to allow this connection too, in addition to telling torsocks to allow inbound connections. Full example: - sudo iptables -I OUTPUT 2 -p tcp -d 127.0.0.1 -m tcp --dport 18081 -j ACCEPT - DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip 127.0.0.1 \ - --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain +```bash +sudo iptables -I OUTPUT 2 -p tcp -d 127.0.0.1 -m tcp --dport 18081 -j ACCEPT +DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip 127.0.0.1 \ + --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain +``` ## Debugging @@ -682,13 +774,13 @@ This section contains general instructions for debugging failed installs or prob We generally use the tool `gdb` (GNU debugger) to provide stack trace functionality, and `ulimit` to provide core dumps in builds which crash or segfault. -* To use gdb in order to obtain a stack trace for a build that has stalled: +* To use `gdb` in order to obtain a stack trace for a build that has stalled: Run the build. Once it stalls, enter the following command: -``` +```bash gdb /path/to/monerod `pidof monerod` ``` @@ -706,11 +798,13 @@ When it terminates with an output along the lines of "Segmentation fault (core d You can now analyse this core dump with `gdb` as follows: -`gdb /path/to/monerod /path/to/dumpfile` +```bash +gdb /path/to/monerod /path/to/dumpfile` +``` Print the stack trace with `bt` -* To run monero within gdb: +#### To run monero within gdb: Type `gdb /path/to/monerod` @@ -722,15 +816,17 @@ Type `run` to run monerod There are two tools available: -* ASAN +#### ASAN Configure Monero with the -D SANITIZE=ON cmake flag, eg: - cd build/debug && cmake -D SANITIZE=ON -D CMAKE_BUILD_TYPE=Debug ../.. +```bash +cd build/debug && cmake -D SANITIZE=ON -D CMAKE_BUILD_TYPE=Debug ../.. +``` You can then run the monero tools normally. Performance will typically halve. -* valgrind +#### valgrind Install valgrind and run as `valgrind /path/to/monerod`. It will be very slow. @@ -740,7 +836,9 @@ Instructions for debugging suspected blockchain corruption as per @HYC There is an `mdb_stat` command in the LMDB source that can print statistics about the database but it's not routinely built. This can be built with the following command: -`cd ~/monero/external/db_drivers/liblmdb && make` +```bash +cd ~/monero/external/db_drivers/liblmdb && make +``` The output of `mdb_stat -ea <path to blockchain dir>` will indicate inconsistencies in the blocks, block_heights and block_info table. diff --git a/cmake/FindBacktrace.cmake b/cmake/FindBacktrace.cmake index 89bbad07c..8137125ab 100644 --- a/cmake/FindBacktrace.cmake +++ b/cmake/FindBacktrace.cmake @@ -83,6 +83,14 @@ else() set(_Backtrace_STD_ARGS Backtrace_LIBRARY ${_Backtrace_STD_ARGS}) endif() +message(STATUS "Backtrace_LIBRARY: ${Backtrace_LIBRARY}") +if(Backtrace_LIBRARY STREQUAL "NOTFOUND") + set(Backtrace_LIBRARY "") +endif() +if(Backtrace_LIBRARY STREQUAL "Backtrace_LIBRARY-NOTFOUND") + set(Backtrace_LIBRARY "") +endif() + set(Backtrace_LIBRARIES ${Backtrace_LIBRARY}) set(Backtrace_HEADER "${_Backtrace_HEADER_TRY}" CACHE STRING "Header providing backtrace(3) facility") diff --git a/cmake/GenVersion.cmake b/cmake/GenVersion.cmake index 1ea5b209c..b2ccfbc34 100644 --- a/cmake/GenVersion.cmake +++ b/cmake/GenVersion.cmake @@ -29,7 +29,7 @@ # Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers # Check what commit we're on -execute_process(COMMAND "${GIT}" rev-parse --short HEAD RESULT_VARIABLE RET OUTPUT_VARIABLE COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND "${GIT}" rev-parse --short=9 HEAD RESULT_VARIABLE RET OUTPUT_VARIABLE COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) if(RET) # Something went wrong, set the version tag to -unknown @@ -38,6 +38,7 @@ if(RET) set(VERSIONTAG "unknown") configure_file("src/version.cpp.in" "${TO}") else() + string(SUBSTRING ${COMMIT} 0 9 COMMIT) message(STATUS "You are currently on commit ${COMMIT}") # Get all the tags diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index afa61b93b..ec0e4cfae 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -22,9 +22,16 @@ host_toolchain:=$(HOST)- endif ifneq ($(DEBUG),) -release_type=debug +release_type=Debug else -release_type=release +release_type=Release +endif + +ifneq ($(TESTS),) +build_tests=ON +release_type=Debug +else +build_tests=OFF endif base_build_dir=$(BASEDIR)/work/build @@ -164,6 +171,8 @@ $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_ -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ -e 's|@debug@|$(DEBUG)|' \ + -e 's|@release_type@|$(release_type)|' \ + -e 's|@build_tests@|$(build_tests)|' \ -e 's|@depends@|$(host_cmake)|' \ -e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\ -e 's|@sdk@|$(SDK_PATH)|'\ @@ -208,4 +217,6 @@ download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win + $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) + .PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/contrib/depends/README.md b/contrib/depends/README.md index c9f8b0783..6a15055b4 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -2,21 +2,29 @@ To build dependencies for the current arch+OS: - make +```bash +make +``` To build for another arch/OS: - make HOST=host-platform-triplet +```bash +make HOST=host-platform-triplet +``` For example: - make HOST=x86_64-w64-mingw32 -j4 +```bash +make HOST=x86_64-w64-mingw32 -j4 +``` A toolchain will be generated that's suitable for plugging into Monero's cmake. In the above example, a dir named x86_64-w64-mingw32 will be created. To use it for Monero: - cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32 +```bash +cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32 +``` Common `host-platform-triplets` for cross compilation are: @@ -31,20 +39,24 @@ No other options are needed, the paths are automatically configured. Dependency Options: The following can be set when running make: make FOO=bar - SOURCES_PATH: downloaded sources will be placed here - BASE_CACHE: built packages will be placed here - SDK_PATH: Path where sdk's can be found (used by OSX) - FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up - DEBUG: disable some optimizations and enable more runtime checking - HOST_ID_SALT: Optional salt to use when generating host package ids - BUILD_ID_SALT: Optional salt to use when generating build package ids +``` +SOURCES_PATH: downloaded sources will be placed here +BASE_CACHE: built packages will be placed here +SDK_PATH: Path where sdk's can be found (used by OSX) +FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up +DEBUG: disable some optimizations and enable more runtime checking +HOST_ID_SALT: Optional salt to use when generating host package ids +BUILD_ID_SALT: Optional salt to use when generating build package ids +``` Additional targets: - download: run 'make download' to fetch all sources without building them - download-osx: run 'make download-osx' to fetch all sources needed for osx builds - download-win: run 'make download-win' to fetch all sources needed for win builds - download-linux: run 'make download-linux' to fetch all sources needed for linux builds +``` +download: run 'make download' to fetch all sources without building them +download-osx: run 'make download-osx' to fetch all sources needed for osx builds +download-win: run 'make download-win' to fetch all sources needed for win builds +download-linux: run 'make download-linux' to fetch all sources needed for linux builds +``` #Darwin (macos) builds: diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index 15e404e42..469144361 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -213,6 +213,14 @@ $(1): | $($(1)_cached_checksum) endef +stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum + +define ext_add_stages +$(foreach stage,$(stages), + $(1)_$(stage): $($(1)_$(stage)) + .PHONY: $(1)_$(stage)) +endef + # These functions create the build targets for each package. They must be # broken down into small steps so that each part is done for all packages # before moving on to the next step. Otherwise, a package's info diff --git a/contrib/depends/packages.md b/contrib/depends/packages.md index 7c8036250..ae5b47327 100644 --- a/contrib/depends/packages.md +++ b/contrib/depends/packages.md @@ -9,39 +9,43 @@ General tips: ## Identifiers Each package is required to define at least these variables: - $(package)_version: - Version of the upstream library or program. If there is no version, a - placeholder such as 1.0 can be used. +``` +$(package)_version: +Version of the upstream library or program. If there is no version, a +placeholder such as 1.0 can be used. - $(package)_download_path: - Location of the upstream source, without the file-name. Usually http or - ftp. +$(package)_download_path: +Location of the upstream source, without the file-name. Usually http or +ftp. - $(package)_file_name: - The upstream source filename available at the download path. +$(package)_file_name: +The upstream source filename available at the download path. - $(package)_sha256_hash: - The sha256 hash of the upstream file +$(package)_sha256_hash: +The sha256 hash of the upstream file +``` These variables are optional: - $(package)_build_subdir: - cd to this dir before running configure/build/stage commands. - - $(package)_download_file: - The file-name of the upstream source if it differs from how it should be - stored locally. This can be used to avoid storing file-names with strange - characters. - - $(package)_dependencies: - Names of any other packages that this one depends on. - - $(package)_patches: - Filenames of any patches needed to build the package - - $(package)_extra_sources: - Any extra files that will be fetched via $(package)_fetch_cmds. These are - specified so that they can be fetched and verified via 'make download'. +``` +$(package)_build_subdir: +cd to this dir before running configure/build/stage commands. + +$(package)_download_file: +The file-name of the upstream source if it differs from how it should be +stored locally. This can be used to avoid storing file-names with strange +characters. + +$(package)_dependencies: +Names of any other packages that this one depends on. + +$(package)_patches: +Filenames of any patches needed to build the package + +$(package)_extra_sources: +Any extra files that will be fetched via $(package)_fetch_cmds. These are +specified so that they can be fetched and verified via 'make download'. +``` ## Build Variables: @@ -49,47 +53,55 @@ After defining the main identifiers, build variables may be added or customized before running the build commands. They should be added to a function called $(package)_set_vars. For example: - define $(package)_set_vars - ... - endef +``` +define $(package)_set_vars +... +endef +``` Most variables can be prefixed with the host, architecture, or both, to make the modifications specific to that case. For example: - Universal: $(package)_cc=gcc - Linux only: $(package)_linux_cc=gcc - x86_64 only: $(package)_x86_64_cc = gcc - x86_64 linux only: $(package)_x86_64_linux_cc = gcc +``` +Universal: $(package)_cc=gcc +Linux only: $(package)_linux_cc=gcc +x86_64 only: $(package)_x86_64_cc = gcc +x86_64 linux only: $(package)_x86_64_linux_cc = gcc +``` These variables may be set to override or append their default values. - $(package)_cc - $(package)_cxx - $(package)_objc - $(package)_objcxx - $(package)_ar - $(package)_ranlib - $(package)_libtool - $(package)_nm - $(package)_cflags - $(package)_cxxflags - $(package)_ldflags - $(package)_cppflags - $(package)_config_env - $(package)_build_env - $(package)_stage_env - $(package)_build_opts - $(package)_config_opts - -The *_env variables are used to add environment variables to the respective +``` +$(package)_cc +$(package)_cxx +$(package)_objc +$(package)_objcxx +$(package)_ar +$(package)_ranlib +$(package)_libtool +$(package)_nm +$(package)_cflags +$(package)_cxxflags +$(package)_ldflags +$(package)_cppflags +$(package)_config_env +$(package)_build_env +$(package)_stage_env +$(package)_build_opts +$(package)_config_opts +``` + +The `*_env` variables are used to add environment variables to the respective commands. Many variables respect a debug/release suffix as well, in order to use them for only the appropriate build config. For example: - $(package)_cflags_release = -O3 - $(package)_cflags_i686_debug = -g - $(package)_config_opts_release = --disable-debug +``` +$(package)_cflags_release = -O3 +$(package)_cflags_i686_debug = -g +$(package)_config_opts_release = --disable-debug +``` These will be used in addition to the options that do not specify debug/release. All builds are considered to be release unless DEBUG=1 is set by @@ -97,51 +109,57 @@ the user. Other variables may be defined as needed. ## Build commands: - For each build, a unique build dir and staging dir are created. For example, - `work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. +For each build, a unique build dir and staging dir are created. For example, +`work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. + +The following build commands are available for each recipe: - The following build commands are available for each recipe: +``` +$(package)_fetch_cmds: +Runs from: build dir +Fetch the source file. If undefined, it will be fetched and verified +against its hash. - $(package)_fetch_cmds: - Runs from: build dir - Fetch the source file. If undefined, it will be fetched and verified - against its hash. +$(package)_extract_cmds: +Runs from: build dir +Verify the source file against its hash and extract it. If undefined, the +source is assumed to be a tarball. - $(package)_extract_cmds: - Runs from: build dir - Verify the source file against its hash and extract it. If undefined, the - source is assumed to be a tarball. +$(package)_preprocess_cmds: +Runs from: build dir/$(package)_build_subdir +Preprocess the source as necessary. If undefined, does nothing. - $(package)_preprocess_cmds: - Runs from: build dir/$(package)_build_subdir - Preprocess the source as necessary. If undefined, does nothing. +$(package)_config_cmds: +Runs from: build dir/$(package)_build_subdir +Configure the source. If undefined, does nothing. - $(package)_config_cmds: - Runs from: build dir/$(package)_build_subdir - Configure the source. If undefined, does nothing. +$(package)_build_cmds: +Runs from: build dir/$(package)_build_subdir +Build the source. If undefined, does nothing. - $(package)_build_cmds: - Runs from: build dir/$(package)_build_subdir - Build the source. If undefined, does nothing. +$(package)_stage_cmds: +Runs from: build dir/$(package)_build_subdir +Stage the build results. If undefined, does nothing. +``` - $(package)_stage_cmds: - Runs from: build dir/$(package)_build_subdir - Stage the build results. If undefined, does nothing. +The following variables are available for each recipe: - The following variables are available for each recipe: - - $(1)_staging_dir: package's destination sysroot path - $(1)_staging_prefix_dir: prefix path inside of the package's staging dir - $(1)_extract_dir: path to the package's extracted sources - $(1)_build_dir: path where configure/build/stage commands will be run - $(1)_patch_dir: path where the package's patches (if any) are found +``` +$(1)_staging_dir: package's destination sysroot path +$(1)_staging_prefix_dir: prefix path inside of the package's staging dir +$(1)_extract_dir: path to the package's extracted sources +$(1)_build_dir: path where configure/build/stage commands will be run +$(1)_patch_dir: path where the package's patches (if any) are found +``` Notes on build commands: -For packages built with autotools, $($(package)_autoconf) can be used in the +For packages built with autotools, `$($(package)_autoconf)` can be used in the configure step to (usually) correctly configure automatically. Any -$($(package)_config_opts) will be appended. +`$($(package)_config_opts`) will be appended. Most autotools projects can be properly staged using: - $(MAKE) DESTDIR=$($(package)_staging_dir) install +```bash +$(MAKE) DESTDIR=$($(package)_staging_dir) install +``` diff --git a/contrib/depends/packages/eudev.mk b/contrib/depends/packages/eudev.mk index e754c0f20..a7795b777 100644 --- a/contrib/depends/packages/eudev.mk +++ b/contrib/depends/packages/eudev.mk @@ -23,3 +23,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/contrib/depends/packages/expat.mk b/contrib/depends/packages/expat.mk index bd2cea1b6..ef81636a2 100644 --- a/contrib/depends/packages/expat.mk +++ b/contrib/depends/packages/expat.mk @@ -6,6 +6,7 @@ $(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd define $(package)_set_vars $(package)_config_opts=--enable-static +$(package)_config_opts=--disable-shared $(package)_config_opts+=--prefix=$(host_prefix) endef @@ -20,3 +21,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/gtest.mk b/contrib/depends/packages/gtest.mk new file mode 100644 index 000000000..5df07a32e --- /dev/null +++ b/contrib/depends/packages/gtest.mk @@ -0,0 +1,38 @@ +package=gtest +$(package)_version=1.8.1 +$(package)_download_path=https://github.com/google/googletest/archive/ +$(package)_file_name=release-$($(package)_version).tar.gz +$(package)_sha256_hash=9bf1fe5182a604b4135edc1a425ae356c9ad15e9b23f9f12a02e80184c3a249c +$(package)_cxxflags=-std=c++11 +$(package)_cxxflags_linux=-fPIC + +define $(package)_config_cmds + cd googletest && \ + CC="$(host_prefix)/native/bin/$($(package)_cc)" \ + CXX="$(host_prefix)/native/bin/$($(package)_cxx)" \ + AR="$(host_prefix)/native/bin/$($(package)_ar)" \ + RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" \ + LIBTOOL="$(host_prefix)/native/bin/$($(package)_libtool)" \ + CXXFLAGS="$($(package)_cxxflags)" \ + CCFLAGS="$($(package)_ccflags)" \ + CPPFLAGS="$($(package)_cppflags)" \ + CFLAGS="$($(package)_cflags) $($(package)_cppflags)" \ + LDLAGS="$($(package)_ldflags)" \ + cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) \ + -DTOOLCHAIN_PREFIX=$(host_toolchain) \ + -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_ar)" \ + -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" \ + -DCMAKE_CXX_FLAGS_DEBUG=ON +endef +# -DCMAKE_TOOLCHAIN_FILE=$(HOST)/share/toolchain.cmake + +define $(package)_build_cmds + cd googletest && CC="$(host_prefix)/native/bin/$($(package)_cc)" $(MAKE) +endef + +define $(package)_stage_cmds + mkdir $($(package)_staging_prefix_dir)/lib $($(package)_staging_prefix_dir)/include &&\ + cp googletest/libgtest.a $($(package)_staging_prefix_dir)/lib/ &&\ + cp googletest/libgtest_main.a $($(package)_staging_prefix_dir)/lib/ &&\ + cp -a googletest/include/* $($(package)_staging_prefix_dir)/include/ +endef diff --git a/contrib/depends/packages/hidapi.mk b/contrib/depends/packages/hidapi.mk index 1c43e525a..a27df04fa 100644 --- a/contrib/depends/packages/hidapi.mk +++ b/contrib/depends/packages/hidapi.mk @@ -1,8 +1,8 @@ package=hidapi -$(package)_version=0.8.0-rc1 -$(package)_download_path=https://github.com/signal11/hidapi/archive +$(package)_version=0.9.0 +$(package)_download_path=https://github.com/libusb/hidapi/archive $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=3c147200bf48a04c1e927cd81589c5ddceff61e6dac137a605f6ac9793f4af61 +$(package)_sha256_hash=630ee1834bdd5c5761ab079fd04f463a89585df8fcae51a7bfe4229b1e02a652 $(package)_linux_dependencies=libusb eudev define $(package)_set_vars @@ -28,3 +28,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/icu4c.mk b/contrib/depends/packages/icu4c.mk index 2b3845488..dc089bbbe 100644 --- a/contrib/depends/packages/icu4c.mk +++ b/contrib/depends/packages/icu4c.mk @@ -21,11 +21,6 @@ define $(package)_config_cmds $(MAKE) $($(package)_build_opts) endef -#define $(package)_build_cmds -# cd source &&\ - $(MAKE) $($((package)_build_opts) `nproc` -#endef - define $(package)_stage_cmds cd buildb &&\ $(MAKE) $($(package)_build_opts) DESTDIR=$($(package)_staging_dir) install lib/* diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index 0b7c3806a..ea4902170 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -6,8 +6,8 @@ $(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f $(package)_dependencies=openssl define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-static --disable-dane-ta-usage --with-drill - $(package)_config_opts=--with-ssl=$(host_prefix) + $(package)_config_opts=--disable-shared --enable-static --with-drill + $(package)_config_opts+=--with-ssl=$(host_prefix) $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic endef @@ -25,4 +25,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + rm lib/*.la endef + diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk index dbcb28141..d4995c1b7 100644 --- a/contrib/depends/packages/libiconv.mk +++ b/contrib/depends/packages/libiconv.mk @@ -28,3 +28,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/contrib/depends/packages/native_cctools.mk b/contrib/depends/packages/native_cctools.mk index 44d238cc4..bcfe1af6b 100644 --- a/contrib/depends/packages/native_cctools.mk +++ b/contrib/depends/packages/native_cctools.mk @@ -52,6 +52,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + cp $($(package)_extract_dir)/cctools/misc/install_name_tool $($(package)_staging_prefix_dir)/bin/ &&\ cd $($(package)_extract_dir)/toolchain && \ mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ diff --git a/contrib/depends/packages/native_cmake-unused.mk b/contrib/depends/packages/native_cmake-unused.mk new file mode 100644 index 000000000..c9ab75711 --- /dev/null +++ b/contrib/depends/packages/native_cmake-unused.mk @@ -0,0 +1,23 @@ +package=native_cmake +$(package)_version=3.14.0 +$(package)_version_dot=v3.14 +$(package)_download_path=https://cmake.org/files/$($(package)_version_dot)/ +$(package)_file_name=cmake-$($(package)_version).tar.gz +$(package)_sha256_hash=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 + +define $(package)_set_vars +$(package)_config_opts= +endef + +define $(package)_config_cmds + ./bootstrap &&\ + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmd + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/contrib/depends/packages/ncurses.mk b/contrib/depends/packages/ncurses.mk new file mode 100644 index 000000000..4e06c00d9 --- /dev/null +++ b/contrib/depends/packages/ncurses.mk @@ -0,0 +1,58 @@ +package=ncurses +$(package)_version=6.1 +$(package)_download_path=https://ftp.gnu.org/gnu/ncurses +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=aa057eeeb4a14d470101eff4597d5833dcef5965331be3528c08d99cebaa0d17 + +define $(package)_set_vars + $(package)_build_opts=CC="$($(package)_cc)" + $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" + $(package)_config_env_darwin=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" + $(package)_config_opts=--prefix=$(host_prefix) + $(package)_config_opts+=--disable-shared + $(package)_config_opts+=--with-build-cc=gcc + $(package)_config_opts+=--without-debug + $(package)_config_opts+=--without-ada + $(package)_config_opts+=--without-cxx-binding + $(package)_config_opts+=--without-cxx + $(package)_config_opts+=--without-ticlib + $(package)_config_opts+=--without-tic + $(package)_config_opts+=--without-progs + $(package)_config_opts+=--without-tests + $(package)_config_opts+=--without-tack + $(package)_config_opts+=--without-manpages + $(package)_config_opts+=--disable-tic-depends + $(package)_config_opts+=--disable-big-strings + $(package)_config_opts+=--disable-ext-colors + $(package)_config_opts+=--enable-pc-files + $(package)_config_opts+=--host=$(HOST) + $(pacakge)_config_opts+=--without-shared + $(pacakge)_config_opts+=--without-pthread + $(pacakge)_config_opts+=--disable-rpath + $(pacakge)_config_opts+=--disable-colorfgbg + $(pacakge)_config_opts+=--disable-ext-colors + $(pacakge)_config_opts+=--disable-ext-mouse + $(pacakge)_config_opts+=--disable-symlinks + $(pacakge)_config_opts+=--enable-warnings + $(pacakge)_config_opts+=--enable-assertions + $(pacakge)_config_opts+=--disable-home-terminfo + $(pacakge)_config_opts+=--enable-database + $(pacakge)_config_opts+=--enable-sp-funcs + $(pacakge)_config_opts+=--enable-term-driver + $(pacakge)_config_opts+=--enable-interop + $(pacakge)_config_opts+=--enable-widec + $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" +endef + +define $(package)_config_cmds + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) $($(package)_build_opts) V=1 +endef + +define $(package)_stage_cmds + $(MAKE) install DESTDIR=$($(package)_staging_dir) +endef + diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index e920b4409..e39dc1d04 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=1.0.2q +$(package)_version=1.0.2r $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=5744cfcbcec2b1b48629f7354203bc1e5e9b5466998bbccc5b5fcde3b18eb684 +$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 1db50580b..562f4f7d6 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,16 +1,16 @@ -packages:=boost openssl zeromq cppzmq expat ldns cppzmq readline libiconv hidapi protobuf libusb +packages:=boost openssl zeromq cppzmq expat ldns libiconv hidapi protobuf libusb native_packages := native_ccache native_protobuf darwin_native_packages = native_biplist native_ds_store native_mac_alias -darwin_packages = sodium-darwin +darwin_packages = sodium-darwin ncurses readline -linux_packages = eudev +linux_packages = eudev ncurses readline unwind sodium qt_packages = qt -ifeq ($(host_os),linux) -packages += unwind -packages += sodium +ifeq ($(build_tests),ON) +packages += gtest endif + ifeq ($(host_os),mingw32) packages += icu4c packages += sodium diff --git a/contrib/depends/packages/protobuf.mk b/contrib/depends/packages/protobuf.mk index 54d3fd924..81fa78a3f 100644 --- a/contrib/depends/packages/protobuf.mk +++ b/contrib/depends/packages/protobuf.mk @@ -25,5 +25,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/libprotoc.a + rm lib/libprotoc.a &&\ + rm lib/*.la endef + diff --git a/contrib/depends/packages/readline.mk b/contrib/depends/packages/readline.mk index afefc7f07..0e2100749 100644 --- a/contrib/depends/packages/readline.mk +++ b/contrib/depends/packages/readline.mk @@ -3,19 +3,19 @@ $(package)_version=8.0 $(package)_download_path=https://ftp.gnu.org/gnu/readline $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 +$(package)_dependencies=ncurses define $(package)_set_vars - $(package)_build_opts=CC="$($(package)_cc)" - $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" $(package)_config_opts=--prefix=$(host_prefix) - $(package)_config_opts+=--disable-shared --enable-multibye --without-purify --without-curses + $(package)_config_opts+=--exec-prefix=$(host_prefix) + $(package)_config_opts+=--host=$(HOST) + $(package)_config_opts+=--disable-shared --with-curses $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_darwin+=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" endef define $(package)_config_cmds - export bash_cv_have_mbstate_t=yes &&\ - export bash_cv_wcwidth_broken=yes &&\ ./configure $($(package)_config_opts) endef @@ -24,6 +24,6 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) install DESTDIR=$($(package)_staging_dir) prefix=$(host_prefix) exec-prefix=$(host_prefix) endef diff --git a/contrib/depends/packages/sodium-darwin.mk b/contrib/depends/packages/sodium-darwin.mk index 8b6ee3f1d..9f11a9426 100644 --- a/contrib/depends/packages/sodium-darwin.mk +++ b/contrib/depends/packages/sodium-darwin.mk @@ -23,3 +23,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk index 06aa8f874..b71f4383e 100644 --- a/contrib/depends/packages/sodium.mk +++ b/contrib/depends/packages/sodium.mk @@ -23,3 +23,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk index 543f868a5..fddbd0561 100644 --- a/contrib/depends/packages/unwind.mk +++ b/contrib/depends/packages/unwind.mk @@ -19,4 +19,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + rm lib/*.la endef + diff --git a/contrib/depends/packages/zeromq.mk b/contrib/depends/packages/zeromq.mk index 01146c26f..f17dbeebe 100644 --- a/contrib/depends/packages/zeromq.mk +++ b/contrib/depends/packages/zeromq.mk @@ -30,5 +30,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf bin share + rm -rf bin share &&\ + rm lib/*.la endef + diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index b0af7bd6b..ee0407a5e 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -1,9 +1,17 @@ # Set the system name, either Darwin, Linux, or Windows SET(CMAKE_SYSTEM_NAME @depends@) -SET(CMAKE_BUILD_TYPE release) +SET(CMAKE_BUILD_TYPE @release_type@) -SET(STATIC true) -SET(UNBOUND_STATIC true) +OPTION(STATIC "Link libraries statically" ON) +OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF) +OPTION(BUILD_TESTS "Build tests." OFF) + +SET(STATIC ON) +SET(UNBOUND_STATIC ON) +SET(ARCH "default") + +SET(BUILD_TESTS @build_tests@) +SET(TREZOR_DEBUG @build_tests@) # where is the target environment SET(CMAKE_FIND_ROOT_PATH @prefix@ /usr) @@ -13,6 +21,8 @@ SET(ENV{PKG_CONFIG_PATH} @prefix@/lib/pkgconfig) SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) SET(Readline_ROOT_DIR @prefix@) +SET(Readline_INCLUDE_DIR @prefix@/include) +SET(Termcap_LIBRARY @prefix@/lib/libncurses.a) SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 374a28a2e..c1aa0fe5f 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -70,7 +70,7 @@ namespace net_utils struct i_connection_filter { - virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address)=0; + virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address, time_t *t = NULL)=0; protected: virtual ~i_connection_filter(){} }; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 821594355..fa5858b9f 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -54,6 +54,9 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +#define AGGRESSIVE_TIMEOUT_THRESHOLD 120 // sockets +#define NEW_CONNECTION_TIMEOUT_LOCAL 1200000 // 2 minutes +#define NEW_CONNECTION_TIMEOUT_REMOTE 10000 // 10 seconds #define DEFAULT_TIMEOUT_MS_LOCAL 1800000 // 30 minutes #define DEFAULT_TIMEOUT_MS_REMOTE 300000 // 5 minutes #define TIMEOUT_EXTRA_MS_PER_BYTE 0.2 @@ -189,7 +192,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_protocol_handler.after_init_connection(); - reset_timer(get_default_timeout(), false); + reset_timer(boost::posix_time::milliseconds(m_local ? NEW_CONNECTION_TIMEOUT_LOCAL : NEW_CONNECTION_TIMEOUT_REMOTE), false); // first read on the raw socket to detect SSL for the server buffer_ssl_init_fill = 0; @@ -691,7 +694,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { unsigned count; try { count = host_count(m_host); } catch (...) { count = 0; } - const unsigned shift = std::min(std::max(count, 1u) - 1, 8u); + const unsigned shift = get_state().sock_count > AGGRESSIVE_TIMEOUT_THRESHOLD ? std::min(std::max(count, 1u) - 1, 8u) : 0; boost::posix_time::milliseconds timeout(0); if (m_local) timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_LOCAL >> shift); @@ -730,8 +733,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add) { - if (m_connection_type != e_connection_type_RPC) - return; MTRACE("Setting " << ms << " expiry"); auto self = safe_shared_from_this(); if(!self) diff --git a/contrib/epee/include/net/net_fwd.h b/contrib/epee/include/net/net_fwd.h new file mode 100644 index 000000000..ba4fe6259 --- /dev/null +++ b/contrib/epee/include/net/net_fwd.h @@ -0,0 +1,38 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace epee +{ + namespace net_utils + { + struct ssl_authentication_t; + class ssl_options_t; + } +} diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 6387c4c34..e315555fc 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -193,7 +193,6 @@ namespace net_utils return CONNECT_FAILURE; } } - m_ssl_options.support = ssl_support_t::e_ssl_support_enabled; } return CONNECT_SUCCESS; }else @@ -223,7 +222,6 @@ namespace net_utils return false; if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { - m_ssl_options.support = epee::net_utils::ssl_support_t::e_ssl_support_enabled; if (try_connect_result == CONNECT_NO_SSL) { MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); @@ -396,7 +394,7 @@ namespace net_utils if (!m_connected || !m_ssl_socket->next_layer().is_open()) return false; if (ssl) - *ssl = m_ssl_options.support == ssl_support_t::e_ssl_support_enabled; + *ssl = m_ssl_options.support != ssl_support_t::e_ssl_support_disabled; return true; } @@ -651,7 +649,7 @@ namespace net_utils bool write(const void* data, size_t sz, boost::system::error_code& ec) { bool success; - if(m_ssl_options.support == ssl_support_t::e_ssl_support_enabled) + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) success = boost::asio::write(*m_ssl_socket, boost::asio::buffer(data, sz), ec); else success = boost::asio::write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), ec); @@ -660,7 +658,7 @@ namespace net_utils void async_write(const void* data, size_t sz, boost::system::error_code& ec) { - if(m_ssl_options.support == ssl_support_t::e_ssl_support_enabled) + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) boost::asio::async_write(*m_ssl_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); else boost::asio::async_write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); @@ -668,7 +666,7 @@ namespace net_utils void async_read(char* buff, size_t sz, boost::asio::detail::transfer_at_least_t transfer_at_least, handler_obj& hndlr) { - if(m_ssl_options.support != ssl_support_t::e_ssl_support_enabled) + if(m_ssl_options.support == ssl_support_t::e_ssl_support_disabled) boost::asio::async_read(m_ssl_socket->next_layer(), boost::asio::buffer(buff, sz), transfer_at_least, hndlr); else boost::asio::async_read(*m_ssl_socket, boost::asio::buffer(buff, sz), transfer_at_least, hndlr); diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 5ef2ff59d..3a97dfdaf 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -135,6 +135,9 @@ namespace net_utils constexpr size_t get_ssl_magic_size() { return 9; } bool is_ssl(const unsigned char *data, size_t len); bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s); + + bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); } } diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 50536f63b..62618537f 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -41,7 +41,7 @@ #define MONERO_DEFAULT_LOG_CATEGORY "net" #ifndef MAKE_IP -#define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) +#define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(((uint32_t)a4)<<24)) #endif #if BOOST_VERSION >= 107000 @@ -107,6 +107,53 @@ namespace net_utils inline bool operator>=(const ipv4_network_address& lhs, const ipv4_network_address& rhs) noexcept { return !lhs.less(rhs); } + class ipv4_network_subnet + { + uint32_t m_ip; + uint8_t m_mask; + + public: + constexpr ipv4_network_subnet() noexcept + : ipv4_network_subnet(0, 0) + {} + + constexpr ipv4_network_subnet(uint32_t ip, uint8_t mask) noexcept + : m_ip(ip), m_mask(mask) {} + + bool equal(const ipv4_network_subnet& other) const noexcept; + bool less(const ipv4_network_subnet& other) const noexcept; + constexpr bool is_same_host(const ipv4_network_subnet& other) const noexcept + { return subnet() == other.subnet(); } + bool matches(const ipv4_network_address &address) const; + + constexpr uint32_t subnet() const noexcept { return m_ip & ~(0xffffffffull << m_mask); } + std::string str() const; + std::string host_str() const; + bool is_loopback() const; + bool is_local() const; + static constexpr address_type get_type_id() noexcept { return address_type::invalid; } + static constexpr zone get_zone() noexcept { return zone::public_; } + static constexpr bool is_blockable() noexcept { return true; } + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(m_ip) + KV_SERIALIZE(m_mask) + END_KV_SERIALIZE_MAP() + }; + + inline bool operator==(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return lhs.equal(rhs); } + inline bool operator!=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return !lhs.equal(rhs); } + inline bool operator<(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return lhs.less(rhs); } + inline bool operator<=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return !rhs.less(lhs); } + inline bool operator>(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return rhs.less(lhs); } + inline bool operator>=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return !lhs.less(rhs); } + class network_address { struct interface @@ -294,6 +341,11 @@ namespace net_utils m_max_speed_up(0) {} + connection_context_base(const connection_context_base& a): connection_context_base() + { + set_details(a.m_connection_id, a.m_remote_address, a.m_is_income, a.m_ssl); + } + connection_context_base& operator=(const connection_context_base& a) { set_details(a.m_connection_id, a.m_remote_address, a.m_is_income, a.m_ssl); diff --git a/contrib/epee/include/rolling_median.h b/contrib/epee/include/rolling_median.h new file mode 100644 index 000000000..8b5a82a84 --- /dev/null +++ b/contrib/epee/include/rolling_median.h @@ -0,0 +1,236 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from source by AShelly: +// Copyright (c) 2011 ashelly.myopenid.com, licenced under the MIT licence +// https://stackoverflow.com/questions/5527437/rolling-median-in-c-turlach-implementation +// https://stackoverflow.com/questions/1309263/rolling-median-algorithm-in-c +// https://ideone.com/XPbl6 + +#pragma once + +#include <stdlib.h> +#include <stdint.h> + +namespace epee +{ +namespace misc_utils +{ + +template<typename Item> +struct rolling_median_t +{ +private: + Item* data; //circular queue of values + int* pos; //index into `heap` for each value + int* heap; //max/median/min heap holding indexes into `data`. + int N; //allocated size. + int idx; //position in circular queue + int minCt; //count of items in min heap + int maxCt; //count of items in max heap + int sz; //count of items in heap + +private: + + //returns true if heap[i] < heap[j] + bool mmless(int i, int j) const + { + return data[heap[i]] < data[heap[j]]; + } + + //swaps items i&j in heap, maintains indexes + bool mmexchange(int i, int j) + { + const int t = heap[i]; + heap[i] = heap[j]; + heap[j] = t; + pos[heap[i]] = i; + pos[heap[j]] = j; + return 1; + } + + //swaps items i&j if i<j; returns true if swapped + bool mmCmpExch(int i, int j) + { + return mmless(i, j) && mmexchange(i, j); + } + + //maintains minheap property for all items below i. + void minSortDown(int i) + { + for (i *= 2; i <= minCt; i *= 2) + { + if (i < minCt && mmless(i + 1, i)) + ++i; + if (!mmCmpExch(i, i / 2)) + break; + } + } + + //maintains maxheap property for all items below i. (negative indexes) + void maxSortDown(int i) + { + for (i *= 2; i >= -maxCt; i *= 2) + { + if (i > -maxCt && mmless(i, i - 1)) + --i; + if (!mmCmpExch(i / 2, i)) + break; + } + } + + //maintains minheap property for all items above i, including median + //returns true if median changed + bool minSortUp(int i) + { + while (i > 0 && mmCmpExch(i, i / 2)) + i /= 2; + return i == 0; + } + + //maintains maxheap property for all items above i, including median + //returns true if median changed + bool maxSortUp(int i) + { + while (i < 0 && mmCmpExch(i / 2, i)) + i /= 2; + return i == 0; + } + +protected: + rolling_median_t &operator=(const rolling_median_t&) = delete; + rolling_median_t(const rolling_median_t&) = delete; + +public: + //creates new rolling_median_t: to calculate `nItems` running median. + rolling_median_t(size_t N): N(N) + { + int size = N * (sizeof(Item) + sizeof(int) * 2); + data = (Item*)malloc(size); + pos = (int*) (data + N); + heap = pos + N + (N / 2); //points to middle of storage. + clear(); + } + + rolling_median_t(rolling_median_t &&m) + { + free(data); + memcpy(this, &m, sizeof(rolling_median_t)); + m.data = NULL; + } + rolling_median_t &operator=(rolling_median_t &&m) + { + free(data); + memcpy(this, &m, sizeof(rolling_median_t)); + m.data = NULL; + return *this; + } + + ~rolling_median_t() + { + free(data); + } + + void clear() + { + idx = 0; + minCt = 0; + maxCt = 0; + sz = 0; + int nItems = N; + while (nItems--) //set up initial heap fill pattern: median,max,min,max,... + { + pos[nItems] = ((nItems + 1) / 2) * ((nItems & 1) ? -1 : 1); + heap[pos[nItems]] = nItems; + } + } + + int size() const + { + return sz; + } + + //Inserts item, maintains median in O(lg nItems) + void insert(Item v) + { + int p = pos[idx]; + Item old = data[idx]; + data[idx] = v; + idx = (idx + 1) % N; + sz = std::min<int>(sz + 1, N); + if (p > 0) //new item is in minHeap + { + if (minCt < (N - 1) / 2) + { + ++minCt; + } + else if (v > old) + { + minSortDown(p); + return; + } + if (minSortUp(p) && mmCmpExch(0, -1)) + maxSortDown(-1); + } + else if (p < 0) //new item is in maxheap + { + if (maxCt < N / 2) + { + ++maxCt; + } + else if (v < old) + { + maxSortDown(p); + return; + } + if (maxSortUp(p) && minCt && mmCmpExch(1, 0)) + minSortDown(1); + } + else //new item is at median + { + if (maxCt && maxSortUp(-1)) + maxSortDown(-1); + if (minCt && minSortUp(1)) + minSortDown(1); + } + } + + //returns median item (or average of 2 when item count is even) + Item median() const + { + Item v = data[heap[0]]; + if (minCt < maxCt) + { + v = (v + data[heap[-1]]) / 2; + } + return v; + } +}; + +} +} diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index 2884f8c5e..e0a32b3ca 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -136,6 +136,7 @@ namespace epee //for pod types array_entry_t<type_name> sa; size_t size = read_varint(); + CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed"); sa.reserve(size); //TODO: add some optimization here later while(size--) diff --git a/contrib/epee/src/buffer.cpp b/contrib/epee/src/buffer.cpp index d637b905e..10ea6de56 100644 --- a/contrib/epee/src/buffer.cpp +++ b/contrib/epee/src/buffer.cpp @@ -64,7 +64,8 @@ void buffer::append(const void *data, size_t sz) size_t reserve = (((size() + sz) * 3 / 2) + 4095) & ~4095; new_storage.reserve(reserve); new_storage.resize(size()); - memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset); + if (size() > 0) + memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset); offset = 0; std::swap(storage, new_storage); } diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index a8bfd114d..4c6ad5516 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -100,7 +100,7 @@ static const char *get_default_categories(int level) switch (level) { case 0: - categories = "*:WARNING,net:FATAL,net.http:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO"; + categories = "*:WARNING,net:FATAL,net.http:FATAL,net.ssl:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO"; break; case 1: categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO,perf.*:DEBUG"; diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index c17d86eca..7d48d2a64 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -78,6 +78,24 @@ namespace }; using openssl_bignum = std::unique_ptr<BIGNUM, openssl_bignum_free>; + struct openssl_ec_key_free + { + void operator()(EC_KEY* ptr) const noexcept + { + EC_KEY_free(ptr); + } + }; + using openssl_ec_key = std::unique_ptr<EC_KEY, openssl_ec_key_free>; + + struct openssl_group_free + { + void operator()(EC_GROUP* ptr) const noexcept + { + EC_GROUP_free(ptr); + } + }; + using openssl_group = std::unique_ptr<EC_GROUP, openssl_group_free>; + boost::system::error_code load_ca_file(boost::asio::ssl::context& ctx, const std::string& path) { SSL_CTX* const ssl_ctx = ctx.native_handle(); // could be moved from context @@ -101,7 +119,7 @@ namespace net_utils // https://stackoverflow.com/questions/256405/programmatically-create-x509-certificate-using-openssl -bool create_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) +bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) { MGINFO("Generating SSL certificate"); pkey = EVP_PKEY_new(); @@ -171,6 +189,87 @@ bool create_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) return true; } +bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert, int type) +{ + MGINFO("Generating SSL certificate"); + pkey = EVP_PKEY_new(); + if (!pkey) + { + MERROR("Failed to create new private key"); + return false; + } + + openssl_pkey pkey_deleter{pkey}; + openssl_ec_key ec_key{EC_KEY_new()}; + if (!ec_key) + { + MERROR("Error allocating EC private key"); + return false; + } + + EC_GROUP *group = EC_GROUP_new_by_curve_name(type); + if (!group) + { + MERROR("Error getting EC group " << type); + return false; + } + openssl_group group_deleter{group}; + + EC_GROUP_set_asn1_flag(group, OPENSSL_EC_NAMED_CURVE); + EC_GROUP_set_point_conversion_form(group, POINT_CONVERSION_UNCOMPRESSED); + + if (!EC_GROUP_check(group, NULL)) + { + MERROR("Group failed check: " << ERR_reason_error_string(ERR_get_error())); + return false; + } + if (EC_KEY_set_group(ec_key.get(), group) != 1) + { + MERROR("Error setting EC group"); + return false; + } + if (EC_KEY_generate_key(ec_key.get()) != 1) + { + MERROR("Error generating EC private key"); + return false; + } + if (EVP_PKEY_assign_EC_KEY(pkey, ec_key.get()) <= 0) + { + MERROR("Error assigning EC private key"); + return false; + } + + // the key is now managed by the EVP_PKEY structure + (void)ec_key.release(); + + cert = X509_new(); + if (!cert) + { + MERROR("Failed to create new X509 certificate"); + return false; + } + ASN1_INTEGER_set(X509_get_serialNumber(cert), 1); + X509_gmtime_adj(X509_get_notBefore(cert), 0); + X509_gmtime_adj(X509_get_notAfter(cert), 3600 * 24 * 182); // half a year + if (!X509_set_pubkey(cert, pkey)) + { + MERROR("Error setting pubkey on certificate"); + X509_free(cert); + return false; + } + X509_NAME *name = X509_get_subject_name(cert); + X509_set_issuer_name(cert, name); + + if (X509_sign(cert, pkey, EVP_sha256()) == 0) + { + MERROR("Error signing certificate"); + X509_free(cert); + return false; + } + (void)pkey_deleter.release(); + return true; +} + ssl_options_t::ssl_options_t(std::vector<std::vector<std::uint8_t>> fingerprints, std::string ca_path) : fingerprints_(std::move(fingerprints)), ca_path(std::move(ca_path)), @@ -195,7 +294,7 @@ boost::asio::ssl::context ssl_options_t::create_context() const ssl_context.set_options(boost::asio::ssl::context::no_tlsv1_1); // only allow a select handful of tls v1.3 and v1.2 ciphers to be used - SSL_CTX_set_cipher_list(ssl_context.native_handle(), "ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-CHACHA20-POLY1305"); + SSL_CTX_set_cipher_list(ssl_context.native_handle(), "ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256"); // set options on the SSL context for added security SSL_CTX *ctx = ssl_context.native_handle(); @@ -214,6 +313,10 @@ boost::asio::ssl::context ssl_options_t::create_context() const #ifdef SSL_OP_NO_COMPRESSION SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION); #endif +#ifdef SSL_OP_CIPHER_SERVER_PREFERENCE + SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); +#endif + SSL_CTX_set_ecdh_auto(ctx, 1); switch (verification) { @@ -240,11 +343,29 @@ boost::asio::ssl::context ssl_options_t::create_context() const { EVP_PKEY *pkey; X509 *cert; - CHECK_AND_ASSERT_THROW_MES(create_ssl_certificate(pkey, cert), "Failed to create certificate"); + bool ok = false; + +#ifdef USE_EXTRA_EC_CERT + CHECK_AND_ASSERT_THROW_MES(create_ec_ssl_certificate(pkey, cert, NID_secp256k1), "Failed to create certificate"); + CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_certificate(ctx, cert), "Failed to use generated certificate"); + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) + MERROR("Failed to use generated EC private key for " << NID_secp256k1); + else + ok = true; + X509_free(cert); + EVP_PKEY_free(pkey); +#endif + + CHECK_AND_ASSERT_THROW_MES(create_rsa_ssl_certificate(pkey, cert), "Failed to create certificate"); CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_certificate(ctx, cert), "Failed to use generated certificate"); - // don't free the cert, the CTX owns it now - CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_PrivateKey(ctx, pkey), "Failed to use generated private key"); + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) + MERROR("Failed to use generated RSA private key for RSA"); + else + ok = true; + X509_free(cert); EVP_PKEY_free(pkey); + + CHECK_AND_ASSERT_THROW_MES(ok, "Failed to use any generated certificate"); } else auth.use_ssl_certificate(ssl_context); diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index 9b781027e..b7f07a23b 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -22,6 +22,24 @@ namespace epee { namespace net_utils bool ipv4_network_address::is_local() const { return net_utils::is_ip_local(ip()); } + bool ipv4_network_subnet::equal(const ipv4_network_subnet& other) const noexcept + { return is_same_host(other) && m_mask == other.m_mask; } + + bool ipv4_network_subnet::less(const ipv4_network_subnet& other) const noexcept + { return subnet() < other.subnet() ? true : (other.subnet() < subnet() ? false : (m_mask < other.m_mask)); } + + std::string ipv4_network_subnet::str() const + { return string_tools::get_ip_string_from_int32(subnet()) + "/" + std::to_string(m_mask); } + + std::string ipv4_network_subnet::host_str() const { return string_tools::get_ip_string_from_int32(subnet()) + "/" + std::to_string(m_mask); } + bool ipv4_network_subnet::is_loopback() const { return net_utils::is_ip_loopback(subnet()); } + bool ipv4_network_subnet::is_local() const { return net_utils::is_ip_local(subnet()); } + bool ipv4_network_subnet::matches(const ipv4_network_address &address) const + { + return (address.ip() & ~(0xffffffffull << m_mask)) == subnet(); + } + + bool network_address::equal(const network_address& other) const { // clang typeid workaround diff --git a/contrib/epee/src/wipeable_string.cpp b/contrib/epee/src/wipeable_string.cpp index 3a6ee5dac..4209b71bf 100644 --- a/contrib/epee/src/wipeable_string.cpp +++ b/contrib/epee/src/wipeable_string.cpp @@ -62,13 +62,15 @@ wipeable_string::wipeable_string(wipeable_string &&other) wipeable_string::wipeable_string(const std::string &other) { grow(other.size()); - memcpy(buffer.data(), other.c_str(), size()); + if (size() > 0) + memcpy(buffer.data(), other.c_str(), size()); } wipeable_string::wipeable_string(std::string &&other) { grow(other.size()); - memcpy(buffer.data(), other.c_str(), size()); + if (size() > 0) + memcpy(buffer.data(), other.c_str(), size()); if (!other.empty()) { memwipe(&other[0], other.size()); // we're kinda left with this again aren't we @@ -79,7 +81,8 @@ wipeable_string::wipeable_string(std::string &&other) wipeable_string::wipeable_string(const char *s) { grow(strlen(s)); - memcpy(buffer.data(), s, size()); + if (size() > 0) + memcpy(buffer.data(), s, size()); } wipeable_string::wipeable_string(const char *s, size_t len) @@ -112,14 +115,18 @@ void wipeable_string::grow(size_t sz, size_t reserved) } size_t old_sz = buffer.size(); std::unique_ptr<char[]> tmp{new char[old_sz]}; - memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); if (old_sz > 0) + { + memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); memwipe(buffer.data(), old_sz * sizeof(char)); + } buffer.reserve(reserved); buffer.resize(sz); - memcpy(buffer.data(), tmp.get(), old_sz * sizeof(char)); if (old_sz > 0) + { + memcpy(buffer.data(), tmp.get(), old_sz * sizeof(char)); memwipe(tmp.get(), old_sz * sizeof(char)); + } } void wipeable_string::push_back(char c) diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index b869bde87..1efc87e0a 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -119,7 +119,7 @@ In order to sign gitian builds on your host machine, which has your PGP key, fork the gitian.sigs repository and clone it on your host machine, or pass the signed assert file back to your build machine. -``` +```bash git clone git@github.com:monero-project/gitian.sigs.git git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git ``` @@ -156,9 +156,9 @@ git push --set-upstream $NAME v0.14.0 ``` ```bash - gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert - gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert - gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert +gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert +gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert +gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert ``` More Build Options diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 67f174fec..6e3ad108e 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -129,6 +129,7 @@ script: | chmod +x ${WRAP_DIR}/${prog} done + git config --global core.abbrev 9 cd monero BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host @@ -153,7 +154,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON - make + make ${MAKEOPTS} DISTNAME=monero-${i} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 7de302353..1cdb85273 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -77,6 +77,7 @@ script: | create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} + git config --global core.abbrev 9 cd monero BASEPREFIX=`pwd`/contrib/depends @@ -100,7 +101,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake - make + make ${MAKEOPTS} DISTNAME=monero-${i} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 1eb558300..d15383905 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -100,6 +100,7 @@ script: | create_per-host_linker_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} + git config --global core.abbrev 9 cd monero BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host @@ -125,7 +126,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake - make + make ${MAKEOPTS} DISTNAME=monero-${i} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip diff --git a/external/miniupnp b/external/miniupnp -Subproject 6b9b73a567e351b844f96c077f7b752ea92e298 +Subproject 4c700e09526a7d546394e85628c57e9490feefa diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index da6d76d97..8a21763c8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -133,7 +133,7 @@ if(NOT IOS) add_subdirectory(blockchain_utilities) endif() -if(CMAKE_BUILD_TYPE STREQUAL Debug) +if(BUILD_DEBUG_UTILITIES) add_subdirectory(debug_utilities) endif() diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 9cc8aaaaf..bb4de3ce6 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -763,6 +763,21 @@ public: virtual void batch_stop() = 0; /** + * @brief aborts a batch transaction + * + * If the subclass implements batching, this function should abort the + * batch it is currently on. + * + * If no batch is in-progress, this function should throw a DB_ERROR. + * This exception may change in the future if it is deemed necessary to + * have a more granular exception type for this scenario. + * + * If any of this cannot be done, the subclass should throw the corresponding + * subclass of DB_EXCEPTION + */ + virtual void batch_abort() = 0; + + /** * @brief sets whether or not to batch transactions * * If the subclass implements batching, this function tells it to begin diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index d17eebee4..78db37b5a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -711,7 +711,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin ++num_blocks_used; } if (my_rtxn) block_rtxn_stop(); - avg_block_size = total_block_size / num_blocks_used; + avg_block_size = total_block_size / (num_blocks_used ? num_blocks_used : 1); MDEBUG("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size); } estim: @@ -774,8 +774,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; bi.bi_weight = block_weight; - bi.bi_diff_hi = (cumulative_difficulty >> 64).convert_to<uint64_t>(); - bi.bi_diff_lo = (cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + bi.bi_diff_hi = ((cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); + bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; if (blk.major_version >= 4) @@ -1081,11 +1081,11 @@ void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, int result = 0; - int num_outputs = amount_output_indices.size(); + size_t num_outputs = amount_output_indices.size(); MDB_val_set(k_tx_id, tx_id); MDB_val v; - v.mv_data = (void *)amount_output_indices.data(); + v.mv_data = num_outputs ? (void *)amount_output_indices.data() : (void*)""; v.mv_size = sizeof(uint64_t) * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); @@ -1960,7 +1960,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) TIME_MEASURE_START(t); - size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0; + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0, commit_counter = 0; uint64_t n_bytes = 0; mdb_txn_safe txn; @@ -2063,6 +2063,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) { MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); ++n_pruned_records; + ++commit_counter; n_bytes += k.mv_size + v.mv_size; result = mdb_cursor_del(c_txs_prunable, 0); if (result) @@ -2072,6 +2073,25 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) result = mdb_cursor_del(c_txs_prunable_tip, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + + if (mode != prune_mode_check && commit_counter >= 4096) + { + MDEBUG("Committing txn at checkpoint..."); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + commit_counter = 0; + } } } } @@ -2141,6 +2161,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) result = mdb_cursor_del(c_txs_prunable, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + ++commit_counter; } } } @@ -2157,6 +2178,34 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) ", seed " << epee::string_tools::to_string_hex(pruning_seed)); } } + + if (mode != prune_mode_check && commit_counter >= 4096) + { + MDEBUG("Committing txn at checkpoint..."); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_val val; + val.mv_size = sizeof(ti); + val.mv_data = (void *)&ti; + result = mdb_cursor_get(c_tx_indices, (MDB_val*)&zerokval, &val, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to restore cursor for tx_indices: ", result).c_str())); + commit_counter = 0; + } } mdb_cursor_close(c_tx_indices); } diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index a32359c6c..ac19fae25 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -54,6 +54,7 @@ public: virtual void unlock() override { } virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0) override { return true; } virtual void batch_stop() override {} + virtual void batch_abort() override {} virtual void set_batch_transactions(bool) override {} virtual void block_wtxn_start() override {} virtual void block_wtxn_stop() override {} diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index ad5963f27..1462e3186 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -79,7 +79,7 @@ LMDB flags (more than one may be specified): ## Examples: -``` +```bash $ monero-blockchain-import --database lmdb#fastest $ monero-blockchain-import --database lmdb#nosync diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 6ff184041..f824d93a6 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -637,6 +637,7 @@ static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total v.mv_size = 2 * sizeof(uint64_t); v.mv_data = (void*)data; dbr = mdb_cursor_put(cur, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to write record for per amount outputs: " + std::string(mdb_strerror(dbr))); mdb_cursor_close(cur); } diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat Binary files differindex adc433522..a7d309753 100644 --- a/src/blocks/checkpoints.dat +++ b/src/blocks/checkpoints.dat diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index e31b96646..11bbe2e24 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -209,6 +209,7 @@ namespace cryptonote ADD_CHECKPOINT(1579000, "7d0d7a2346373afd41ed1e744a939fc5d474a7dbaa257be5c6fff4009e789241"); ADD_CHECKPOINT(1668900, "ac2dcaf3d2f58ffcf8391639f0f1ebafcb8eac43c49479c7c37f611868d07568"); ADD_CHECKPOINT(1775600, "1c6e01c661dc22cab939e79ec6a5272190624ce8356d2f7b958e4f9a57fdb05e"); + ADD_CHECKPOINT(1856000, "9b57f17f29c71a3acd8a7904b93c41fa6eb8d2b7c73936ce4f1702d14880ba29"); return true; } diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 5e03bf897..dc1f335a7 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -48,7 +48,6 @@ static const char *DEFAULT_DNS_PUBLIC_ADDR[] = "80.67.169.40", // FDN (France) "89.233.43.71", // http://censurfridns.dk (Denmark) "109.69.8.51", // punCAT (Spain) - "77.109.148.137", // Xiala.net (Switzerland) "193.58.251.251", // SkyDNS (Russia) }; diff --git a/src/common/password.cpp b/src/common/password.cpp index 03d13db42..33e1f48fd 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -57,7 +57,7 @@ namespace DWORD mode_old; ::GetConsoleMode(h_cin, &mode_old); - DWORD mode_new = mode_old & ~(hide_input ? ENABLE_ECHO_INPUT : 0); + DWORD mode_new = mode_old & ~((hide_input ? ENABLE_ECHO_INPUT : 0) | ENABLE_LINE_INPUT); ::SetConsoleMode(h_cin, mode_new); bool r = true; @@ -77,10 +77,6 @@ namespace } else if (ucs2_ch == L'\r') { - continue; - } - else if (ucs2_ch == L'\n') - { std::cout << std::endl; break; } diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index cb5f79da8..dab3e562d 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -36,6 +36,7 @@ #include "storages/http_abstract_invoke.h" #include "net/http_auth.h" #include "net/http_client.h" +#include "net/net_ssl.h" #include "string_tools.h" namespace tools @@ -49,11 +50,12 @@ namespace tools uint32_t ip , uint16_t port , boost::optional<epee::net_utils::http::login> user + , epee::net_utils::ssl_options_t ssl_options ) : m_http_client{} { m_http_client.set_server( - epee::string_tools::get_ip_string_from_int32(ip), std::to_string(port), std::move(user) + epee::string_tools::get_ip_string_from_int32(ip), std::to_string(port), std::move(user), std::move(ssl_options) ); } diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index 170911262..851c70a25 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -105,9 +105,12 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) memset(st, 0, sizeof(st)); for ( ; inlen >= rsiz; inlen -= rsiz, in += rsiz) { - for (i = 0; i < rsizw; i++) - st[i] ^= swap64le(((uint64_t *) in)[i]); - keccakf(st, KECCAK_ROUNDS); + for (i = 0; i < rsizw; i++) { + uint64_t ina; + memcpy(&ina, in + i * 8, 8); + st[i] ^= swap64le(ina); + } + keccakf(st, KECCAK_ROUNDS); } // last block and padding @@ -116,7 +119,8 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) local_abort("Bad keccak use"); } - memcpy(temp, in, inlen); + if (inlen > 0) + memcpy(temp, in, inlen); temp[inlen++] = 1; memset(temp + inlen, 0, rsiz - inlen); temp[rsiz - 1] |= 0x80; diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 7f36c9dc3..13835c823 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -897,7 +897,6 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int // locals to avoid constant TLS dereferencing uint8_t *local_hp_state = hp_state; - v4_random_math_JIT_func local_hp_jitfunc = hp_jitfunc; /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 7802fb67f..0a5860f3b 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -30,6 +30,7 @@ #include <assert.h> #include <stddef.h> +#include <stdlib.h> #include <string.h> #include "hash-ops.h" @@ -82,23 +83,24 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { size_t cnt = tree_hash_cnt( count ); - char ints[cnt][HASH_SIZE]; - memset(ints, 0 , sizeof(ints)); // zero out as extra protection for using uninitialized mem + char *ints = calloc(cnt, HASH_SIZE); // zero out as extra protection for using uninitialized mem + assert(ints); memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { - cn_fast_hash(hashes[i], 64, ints[j]); + cn_fast_hash(hashes[i], 64, ints + j * HASH_SIZE); } assert(i == count); while (cnt > 2) { cnt >>= 1; for (i = 0, j = 0; j < cnt; i += 2, ++j) { - cn_fast_hash(ints[i], 64, ints[j]); + cn_fast_hash(ints + i * HASH_SIZE, 64, ints + j * HASH_SIZE); } } - cn_fast_hash(ints[0], 64, root_hash); + cn_fast_hash(ints, 64, root_hash); + free(ints); } } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 20d92bdf1..055c4a22b 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -320,7 +320,7 @@ namespace cryptonote } if (!typename Archive<W>::is_saving()) pruned = true; - return true; + return ar.stream().good(); } private: diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 3dd98f0c6..79ce610a9 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -365,11 +365,11 @@ namespace boost else { // store high part - cryptonote::difficulty_type x_ = x >> 64; + cryptonote::difficulty_type x_ = (x >> 64) & 0xffffffffffffffff; uint64_t v = x_.convert_to<uint64_t>(); a & v; // store low part - x_ = x << 64 >> 64; + x_ = x & 0xffffffffffffffff; v = x_.convert_to<uint64_t>(); a & v; } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 566622c1a..7d7de416d 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -221,8 +221,7 @@ namespace cryptonote tx.invalidate_hashes(); //TODO: validate tx - get_transaction_hash(tx, tx_hash); - return true; + return get_transaction_hash(tx, tx_hash); } //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) @@ -975,6 +974,7 @@ namespace cryptonote { crypto::hash h = null_hash; get_transaction_hash(t, h, NULL); + CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(t, h, NULL), "Failed to calculate transaction hash"); return h; } //--------------------------------------------------------------- @@ -1327,7 +1327,7 @@ namespace cryptonote txs_ids.reserve(1 + b.tx_hashes.size()); crypto::hash h = null_hash; size_t bl_sz = 0; - get_transaction_hash(b.miner_tx, h, bl_sz); + CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(b.miner_tx, h, bl_sz), "Failed to calculate transaction hash"); txs_ids.push_back(h); for(auto& th: b.tx_hashes) txs_ids.push_back(th); diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 173679e21..e594eb049 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -123,7 +123,7 @@ namespace cryptonote m_miner_extra_sleep(BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS), m_block_reward(0) { - + m_attrs.set_stack_size(THREAD_STACK_SIZE); } //----------------------------------------------------------------------------------------------------- miner::~miner() @@ -360,7 +360,7 @@ namespace cryptonote return m_threads_total; } //----------------------------------------------------------------------------------------------------- - bool miner::start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background, bool ignore_battery) + bool miner::start(const account_public_address& adr, size_t threads_count, bool do_background, bool ignore_battery) { m_block_reward = 0; m_mine_address = adr; @@ -371,7 +371,6 @@ namespace cryptonote m_threads_autodetect.push_back({epee::misc_utils::get_ns_count(), m_total_hashes}); m_threads_total = 1; } - m_attrs = attrs; m_starter_nonce = crypto::rand<uint32_t>(); CRITICAL_REGION_LOCAL(m_threads_lock); if(is_mining()) @@ -395,7 +394,7 @@ namespace cryptonote for(size_t i = 0; i != m_threads_total; i++) { - m_threads.push_back(boost::thread(attrs, boost::bind(&miner::worker_thread, this))); + m_threads.push_back(boost::thread(m_attrs, boost::bind(&miner::worker_thread, this))); } if (threads_count == 0) @@ -405,7 +404,7 @@ namespace cryptonote if( get_is_background_mining_enabled() ) { - m_background_mining_thread = boost::thread(attrs, boost::bind(&miner::background_worker_thread, this)); + m_background_mining_thread = boost::thread(m_attrs, boost::bind(&miner::background_worker_thread, this)); LOG_PRINT_L0("Background mining controller thread started" ); } @@ -487,10 +486,7 @@ namespace cryptonote { if(m_do_mining) { - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - start(m_mine_address, m_threads_total, attrs, get_is_background_mining_enabled(), get_ignore_battery()); + start(m_mine_address, m_threads_total, get_is_background_mining_enabled(), get_ignore_battery()); } } //----------------------------------------------------------------------------------------------------- diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 285075f51..ac7a0381c 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -64,7 +64,7 @@ namespace cryptonote static void init_options(boost::program_options::options_description& desc); bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height, uint64_t block_reward); bool on_block_chain_update(); - bool start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background = false, bool ignore_battery = false); + bool start(const account_public_address& adr, size_t threads_count, bool do_background = false, bool ignore_battery = false); uint64_t get_speed() const; uint32_t get_threads_count() const; void send_stop_signal(); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index f246d2f4e..fbff8d878 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -179,9 +179,11 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_long_term_block_weights_window(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), m_long_term_effective_median_block_weight(0), m_long_term_block_weights_cache_tip_hash(crypto::null_hash), + m_long_term_block_weights_cache_rolling_median(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), - m_btc_valid(false) + m_btc_valid(false), + m_batch_success(true) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -519,7 +521,10 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline } if (test_options && test_options->long_term_block_weight_window) + { m_long_term_block_weights_window = test_options->long_term_block_weight_window; + m_long_term_block_weights_cache_rolling_median = epee::misc_utils::rolling_median_t<uint64_t>(m_long_term_block_weights_window); + } { db_txn_guard txn_guard(m_db, m_db->is_read_only()); @@ -615,17 +620,13 @@ void Blockchain::pop_blocks(uint64_t nblocks) CRITICAL_REGION_LOCAL(m_tx_pool); CRITICAL_REGION_LOCAL1(m_blockchain_lock); - while (!m_db->batch_start()) - { - m_blockchain_lock.unlock(); - m_tx_pool.unlock(); - epee::misc_utils::sleep_no_w(1000); - m_tx_pool.lock(); - m_blockchain_lock.lock(); - } + bool stop_batch = m_db->batch_start(); try { + const uint64_t blockchain_height = m_db->height(); + if (blockchain_height > 0) + nblocks = std::min(nblocks, blockchain_height - 1); for (i=0; i < nblocks; ++i) { pop_block_from_blockchain(); @@ -633,10 +634,14 @@ void Blockchain::pop_blocks(uint64_t nblocks) } catch (const std::exception& e) { - LOG_ERROR("Error when popping blocks, only " << i << " blocks are popped: " << e.what()); + LOG_ERROR("Error when popping blocks after processing " << i << " blocks: " << e.what()); + if (stop_batch) + m_db->batch_abort(); + return; } - m_db->batch_stop(); + if (stop_batch) + m_db->batch_stop(); } //------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the @@ -1292,21 +1297,20 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_ weights = m_db->get_block_weights(start_offset, count); } //------------------------------------------------------------------ -void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const +uint64_t Blockchain::get_long_term_block_weight_median(uint64_t start_height, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); PERF_TIMER(get_long_term_block_weights); - if (count == 0) - return; + CHECK_AND_ASSERT_THROW_MES(count > 0, "count == 0"); bool cached = false; uint64_t blockchain_height = m_db->height(); uint64_t tip_height = start_height + count - 1; crypto::hash tip_hash = crypto::null_hash; - if (tip_height < blockchain_height && count == m_long_term_block_weights_cache.size()) + if (tip_height < blockchain_height && count == (size_t)m_long_term_block_weights_cache_rolling_median.size()) { tip_hash = m_db->get_block_hash_from_height(tip_height); cached = tip_hash == m_long_term_block_weights_cache_tip_hash; @@ -1315,32 +1319,30 @@ void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uin if (cached) { MTRACE("requesting " << count << " from " << start_height << ", cached"); - weights = m_long_term_block_weights_cache; - return; + return m_long_term_block_weights_cache_rolling_median.median(); } // in the vast majority of uncached cases, most is still cached, // as we just move the window one block up: - if (tip_height > 0 && count == m_long_term_block_weights_cache.size() && tip_height < blockchain_height) + if (tip_height > 0 && count == (size_t)m_long_term_block_weights_cache_rolling_median.size() && tip_height < blockchain_height) { crypto::hash old_tip_hash = m_db->get_block_hash_from_height(tip_height - 1); if (old_tip_hash == m_long_term_block_weights_cache_tip_hash) { - weights = m_long_term_block_weights_cache; - for (size_t i = 1; i < weights.size(); ++i) - weights[i - 1] = weights[i]; MTRACE("requesting " << count << " from " << start_height << ", incremental"); - weights.back() = m_db->get_block_long_term_weight(tip_height); - m_long_term_block_weights_cache = weights; m_long_term_block_weights_cache_tip_hash = tip_hash; - return; + m_long_term_block_weights_cache_rolling_median.insert(m_db->get_block_long_term_weight(tip_height)); + return m_long_term_block_weights_cache_rolling_median.median(); } } MTRACE("requesting " << count << " from " << start_height << ", uncached"); - weights = m_db->get_long_term_block_weights(start_height, count); - m_long_term_block_weights_cache = weights; + std::vector<uint64_t> weights = m_db->get_long_term_block_weights(start_height, count); m_long_term_block_weights_cache_tip_hash = tip_hash; + m_long_term_block_weights_cache_rolling_median.clear(); + for (uint64_t w: weights) + m_long_term_block_weights_cache_rolling_median.insert(w); + return m_long_term_block_weights_cache_rolling_median.median(); } //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_block_weight_limit() const @@ -1381,7 +1383,8 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, // just as we compare it, we'll just use a slightly old template, but // this would be the case anyway if we'd lock, and the change happened // just after the block template was created - if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce && m_btc_pool_cookie == m_tx_pool.cookie()) { + if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce + && m_btc_pool_cookie == m_tx_pool.cookie() && m_btc.prev_id == get_tail_id()) { MDEBUG("Using cached template"); m_btc.timestamp = time(NULL); // update timestamp unconditionally b = m_btc; @@ -1917,10 +1920,14 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO if (missed_tx_ids.size() != 0) { - LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size() - << " transactions for block with hash: " << get_block_hash(bl.second) - << std::endl - ); + // do not display an error if the peer asked for an unpruned block which we are not meant to have + if (tools::has_unpruned_block(get_block_height(bl.second), get_current_blockchain_height(), get_blockchain_pruning_seed())) + { + LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size() + << " transactions for block with hash: " << get_block_hash(bl.second) + << std::endl + ); + } // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested @@ -2068,7 +2075,6 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, if (to_height > 0 && to_height < from_height) return false; - const uint64_t real_start_height = start_height; if (from_height > start_height) start_height = from_height; @@ -2082,7 +2088,7 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, { std::vector<uint64_t> heights; heights.reserve(to_height + 1 - start_height); - uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height; + const uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height; for (uint64_t h = real_start_height; h <= to_height; ++h) heights.push_back(h); distribution = m_db->get_block_cumulative_rct_outputs(heights); @@ -2360,8 +2366,8 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc if (result) { cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); - resp.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); - resp.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); + resp.cumulative_difficulty = (wide_cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); + resp.cumulative_difficulty_top64 = ((wide_cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); } return result; @@ -3283,7 +3289,6 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); - const uint64_t blockchain_height = m_db->height(); uint64_t median = 0; uint64_t already_generated_coins = 0; @@ -3879,6 +3884,7 @@ leave: catch (const KEY_IMAGE_EXISTS& e) { LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + m_batch_success = false; bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; @@ -3887,6 +3893,7 @@ leave: { //TODO: figure out the best way to deal with this failure LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + m_batch_success = false; bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; @@ -3970,9 +3977,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) return block_weight; - std::vector<uint64_t> weights; - get_long_term_block_weights(weights, db_height - nblocks, nblocks); - uint64_t long_term_median = epee::misc_utils::median(weights); + uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks); uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); uint64_t short_term_constraint = long_term_effective_median_block_weight + long_term_effective_median_block_weight * 2 / 5; @@ -3991,20 +3996,17 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti const uint64_t db_height = m_db->height(); const uint8_t hf_version = get_current_hard_fork_version(); uint64_t full_reward_zone = get_min_block_weight(hf_version); - uint64_t long_term_block_weight; if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) { std::vector<uint64_t> weights; get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); m_current_block_cumul_weight_median = epee::misc_utils::median(weights); - long_term_block_weight = weights.back(); } else { const uint64_t block_weight = m_db->get_block_weight(db_height - 1); - std::vector<uint64_t> weights, new_weights; uint64_t long_term_median; if (db_height == 1) { @@ -4015,23 +4017,27 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); if (nblocks == db_height) --nblocks; - get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks); - new_weights = weights; - long_term_median = epee::misc_utils::median(weights); + long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks); } m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); uint64_t short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; - long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint); + uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint); - if (new_weights.empty()) - new_weights.resize(1); - new_weights[0] = long_term_block_weight; - long_term_median = epee::misc_utils::median(new_weights); + if (db_height == 1) + { + long_term_median = long_term_block_weight; + } + else + { + m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1); + m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight); + long_term_median = m_long_term_block_weights_cache_rolling_median.median(); + } m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); - weights.clear(); + std::vector<uint64_t> weights; get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); uint64_t short_term_median = epee::misc_utils::median(weights); @@ -4195,7 +4201,10 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) try { - m_db->batch_stop(); + if (m_batch_success) + m_db->batch_stop(); + else + m_db->batch_abort(); success = true; } catch (const std::exception &e) @@ -4419,6 +4428,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete m_tx_pool.lock(); m_blockchain_lock.lock(); } + m_batch_success = true; const uint64_t height = m_db->height(); if ((height + blocks_entry.size()) < m_blocks_hash_check.size()) @@ -4895,7 +4905,7 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "570ce2357b08fadac6058e34f95c5e08323f9325de260d07b091a281a948a7b0"; +static const char expected_block_hashes_hash[] = "7dafb40b414a0e59bfced6682ef519f0b416bc914dd3d622b72e0dd1a47117c2"; void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { if (get_checkpoints == nullptr || !m_fast_sync) diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index b7d3ce79b..b1c23b704 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -37,7 +37,6 @@ #include <boost/multi_index/global_fun.hpp> #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/member.hpp> -#include <boost/circular_buffer.hpp> #include <atomic> #include <functional> #include <unordered_map> @@ -46,6 +45,7 @@ #include "span.h" #include "syncobj.h" #include "string_tools.h" +#include "rolling_median.h" #include "cryptonote_basic/cryptonote_basic.h" #include "common/util.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" @@ -1044,7 +1044,7 @@ namespace cryptonote uint64_t m_long_term_block_weights_window; uint64_t m_long_term_effective_median_block_weight; mutable crypto::hash m_long_term_block_weights_cache_tip_hash; - mutable std::vector<uint64_t> m_long_term_block_weights_cache; + mutable epee::misc_utils::rolling_median_t<uint64_t> m_long_term_block_weights_cache_rolling_median; epee::critical_section m_difficulty_lock; crypto::hash m_difficulty_for_next_block_top_hash; @@ -1079,6 +1079,9 @@ namespace cryptonote uint64_t m_btc_expected_reward; bool m_btc_valid; + + bool m_batch_success; + std::shared_ptr<tools::Notify> m_block_notify; std::shared_ptr<tools::Notify> m_reorg_notify; @@ -1291,15 +1294,16 @@ namespace cryptonote void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const; /** - * @brief gets recent block long term weights for median calculation + * @brief gets block long term weight median * - * get the block long term weights of the last <count> blocks, and return by reference <weights>. + * get the block long term weight median of <count> blocks starting at <start_height> * - * @param weights return-by-reference the list of weights * @param start_height the block height of the first block to query * @param count the number of blocks to get weights for + * + * @return the long term median block weight */ - void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const; + uint64_t get_long_term_block_weight_median(uint64_t start_height, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index b678f3e80..0f69f3620 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -453,6 +453,7 @@ namespace cryptonote m_nettype = FAKECHAIN; } bool r = handle_command_line(vm); + CHECK_AND_ASSERT_MES(r, false, "Failed to handle command line"); std::string db_type = command_line::get_arg(vm, cryptonote::arg_db_type); std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); @@ -641,6 +642,7 @@ namespace cryptonote }; const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty, get_checkpoints); + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); @@ -684,9 +686,15 @@ namespace cryptonote if (prune_blockchain) { // display a message if the blockchain is not pruned yet - if (m_blockchain_storage.get_current_blockchain_height() > 1 && !m_blockchain_storage.get_blockchain_pruning_seed()) + if (!m_blockchain_storage.get_blockchain_pruning_seed()) + { MGINFO("Pruning blockchain..."); - CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + } + else + { + CHECK_AND_ASSERT_MES(m_blockchain_storage.update_blockchain_pruning(), false, "Failed to update blockchain pruning"); + } } return load_state_data(); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index c1cbe2acd..49d5a8ccc 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -95,13 +95,17 @@ namespace cryptonote // the whole prepare/handle/cleanup incoming block sequence. class LockedTXN { public: - LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false) { + LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false), m_active(false) { m_batch = m_blockchain.get_db().batch_start(); + m_active = true; } - ~LockedTXN() { try { if (m_batch) { m_blockchain.get_db().batch_stop(); } } catch (const std::exception &e) { MWARNING("LockedTXN dtor filtering exception: " << e.what()); } } + void commit() { try { if (m_batch && m_active) { m_blockchain.get_db().batch_stop(); m_active = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::commit filtering exception: " << e.what()); } } + void abort() { try { if (m_batch && m_active) { m_blockchain.get_db().batch_abort(); m_active = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::abort filtering exception: " << e.what()); } } + ~LockedTXN() { abort(); } private: Blockchain &m_blockchain; bool m_batch; + bool m_active; }; } //--------------------------------------------------------------------------------- @@ -255,6 +259,7 @@ namespace cryptonote if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); + lock.commit(); } catch (const std::exception &e) { @@ -299,6 +304,7 @@ namespace cryptonote if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); + lock.commit(); } catch (const std::exception &e) { @@ -398,6 +404,7 @@ namespace cryptonote return; } } + lock.commit(); if (changed) ++m_cookie; if (m_txpool_weight > bytes) @@ -494,6 +501,7 @@ namespace cryptonote m_blockchain.remove_txpool_tx(id); m_txpool_weight -= tx_weight; remove_transaction_keyimages(tx, id); + lock.commit(); } catch (const std::exception &e) { @@ -578,6 +586,7 @@ namespace cryptonote // ignore error } } + lock.commit(); ++m_cookie; } return true; @@ -641,6 +650,7 @@ namespace cryptonote // continue } } + lock.commit(); } //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_transactions_count(bool include_unrelayed_txes) const @@ -1119,6 +1129,7 @@ namespace cryptonote } } } + lock.commit(); if (changed) ++m_cookie; } @@ -1271,6 +1282,7 @@ namespace cryptonote append_key_images(k_images, tx); LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } + lock.commit(); expected_reward = best_coinbase; LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " @@ -1336,6 +1348,7 @@ namespace cryptonote // continue } } + lock.commit(); } if (n_removed > 0) ++m_cookie; @@ -1395,6 +1408,7 @@ namespace cryptonote // ignore error } } + lock.commit(); } m_cookie = 0; diff --git a/src/cryptonote_core/tx_sanity_check.cpp b/src/cryptonote_core/tx_sanity_check.cpp index 10198a3d3..36be91f2c 100644 --- a/src/cryptonote_core/tx_sanity_check.cpp +++ b/src/cryptonote_core/tx_sanity_check.cpp @@ -88,7 +88,7 @@ bool tx_sanity_check(Blockchain &blockchain, const cryptonote::blobdata &tx_blob std::vector<uint64_t> offsets(rct_indices.begin(), rct_indices.end()); uint64_t median = epee::misc_utils::median(offsets); - if (median < n_available * 9 / 10) + if (median < n_available * 6 / 10) { MERROR("median is " << median << "/" << n_available); return false; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 0927b5d7f..dcc5ec6ed 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -52,6 +52,7 @@ PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) #define LOCALHOST_INT 2130706433 +#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 500 namespace cryptonote { diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 7adca3158..03d04a074 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -341,6 +341,11 @@ namespace cryptonote if(m_core.have_block(hshd.top_id)) { + if (target > hshd.current_height) + { + MINFO(context << "peer is not ahead of us and we're syncing, disconnecting"); + return false; + } context.m_state = cryptonote_connection_context::state_normal; if(is_inital && target == m_core.get_current_blockchain_height()) on_connection_synchronized(); @@ -410,8 +415,8 @@ namespace cryptonote m_core.get_blockchain_top(hshd.current_height, hshd.top_id); hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); difficulty_type wide_cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); - hshd.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); - hshd.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); + hshd.cumulative_difficulty = (wide_cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); + hshd.cumulative_difficulty_top64 = ((wide_cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); hshd.current_height +=1; hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; @@ -809,12 +814,27 @@ namespace cryptonote NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response; fluffy_response.b.block = t_serializable_object_to_blob(b); fluffy_response.current_blockchain_height = arg.current_blockchain_height; + std::vector<bool> seen(b.tx_hashes.size(), false); for(auto& tx_idx: arg.missing_tx_indices) { if(tx_idx < b.tx_hashes.size()) { MDEBUG(" tx " << b.tx_hashes[tx_idx]); + if (seen[tx_idx]) + { + LOG_ERROR_CCONTEXT + ( + "Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX" + << ", request is asking for duplicate tx " + << ", tx index = " << tx_idx << ", block tx count " << b.tx_hashes.size() + << ", block_height = " << arg.current_blockchain_height + << ", dropping connection" + ); + drop_connection(context, true, false); + return 1; + } txids.push_back(b.tx_hashes[tx_idx]); + seen[tx_idx] = true; } else { @@ -914,6 +934,17 @@ namespace cryptonote int t_cryptonote_protocol_handler<t_core>::handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + + if (arg.blocks.size() + arg.txs.size() > CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT) + { + LOG_ERROR_CCONTEXT( + "Requested objects count is too big (" + << arg.blocks.size() + arg.txs.size() << ") expected not more then " + << CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT); + drop_connection(context, false, false); + return 1; + } + NOTIFY_RESPONSE_GET_OBJECTS::request rsp; if(!m_core.handle_get_objects(arg, rsp, context)) { diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 17b945c9a..ed799096b 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -40,10 +40,11 @@ t_command_parser_executor::t_command_parser_executor( uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) - : m_executor(ip, port, login, is_rpc, rpc_server) + : m_executor(ip, port, login, ssl_options, is_rpc, rpc_server) {} bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& args) @@ -595,6 +596,13 @@ bool t_command_parser_executor::unban(const std::vector<std::string>& args) return m_executor.unban(ip); } +bool t_command_parser_executor::banned(const std::vector<std::string>& args) +{ + if (args.size() != 1) return false; + std::string address = args[0]; + return m_executor.banned(address); +} + bool t_command_parser_executor::flush_txpool(const std::vector<std::string>& args) { if (args.size() > 1) return false; diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index 098018642..aeaf4d254 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -40,6 +40,7 @@ #include "daemon/rpc_command_executor.h" #include "common/common_fwd.h" +#include "net/net_fwd.h" #include "rpc/core_rpc_server.h" namespace daemonize { @@ -53,6 +54,7 @@ public: uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server = NULL ); @@ -125,6 +127,8 @@ public: bool unban(const std::vector<std::string>& args); + bool banned(const std::vector<std::string>& args); + bool flush_txpool(const std::vector<std::string>& args); bool output_histogram(const std::vector<std::string>& args); diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 69ad6ff10..f0b4b4ba0 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -43,10 +43,11 @@ t_command_server::t_command_server( uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) - : m_parser(ip, port, login, is_rpc, rpc_server) + : m_parser(ip, port, login, ssl_options, is_rpc, rpc_server) , m_command_lookup() , m_is_rpc(is_rpc) { @@ -242,10 +243,16 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "unban" , std::bind(&t_command_parser_executor::unban, &m_parser, p::_1) - , "unban <IP>" + , "unban <address>" , "Unban a given <IP>." ); m_command_lookup.set_handler( + "banned" + , std::bind(&t_command_parser_executor::banned, &m_parser, p::_1) + , "banned <address>" + , "Check whether an <address> is banned." + ); + m_command_lookup.set_handler( "flush_txpool" , std::bind(&t_command_parser_executor::flush_txpool, &m_parser, p::_1) , "flush_txpool [<txid>]" diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h index c8e77f551..da532223e 100644 --- a/src/daemon/command_server.h +++ b/src/daemon/command_server.h @@ -43,6 +43,7 @@ Passing RPC commands: #include "common/common_fwd.h" #include "console_handler.h" #include "daemon/command_parser_executor.h" +#include "net/net_fwd.h" namespace daemonize { @@ -57,6 +58,7 @@ public: uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc = true , cryptonote::core_rpc_server* rpc_server = NULL ); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 531c080de..5084b6283 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -45,6 +45,7 @@ #include "daemon/command_server.h" #include "daemon/command_server.h" #include "daemon/command_line_args.h" +#include "net/net_ssl.h" #include "version.h" using namespace epee; @@ -163,7 +164,7 @@ bool t_daemon::run(bool interactive) if (interactive && mp_internals->rpcs.size()) { // The first three variables are not used when the fourth is false - rpc_commands.reset(new daemonize::t_command_server(0, 0, boost::none, false, mp_internals->rpcs.front()->get_server())); + rpc_commands.reset(new daemonize::t_command_server(0, 0, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_disabled, false, mp_internals->rpcs.front()->get_server())); rpc_commands->start_handling(std::bind(&daemonize::t_daemon::stop_p2p, this)); } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index dbbb2308c..690d4d60e 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -324,7 +324,11 @@ int main(int argc, char const * argv[]) } } - daemonize::t_command_server rpc_commands{rpc_ip, rpc_port, std::move(login)}; + auto ssl_options = cryptonote::rpc_args::process_ssl(vm, true); + if (!ssl_options) + return 1; + + daemonize::t_command_server rpc_commands{rpc_ip, rpc_port, std::move(login), std::move(*ssl_options)}; if (rpc_commands.process_command_vec(command)) { return 0; diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 186296dc9..bf4b3b09e 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -127,6 +127,7 @@ t_rpc_command_executor::t_rpc_command_executor( uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) @@ -137,7 +138,7 @@ t_rpc_command_executor::t_rpc_command_executor( boost::optional<epee::net_utils::http::login> http_login{}; if (login) http_login.emplace(login->username, login->password.password()); - m_rpc_client = new tools::t_rpc_client(ip, port, std::move(http_login)); + m_rpc_client = new tools::t_rpc_client(ip, port, std::move(http_login), ssl_options); } else { @@ -722,7 +723,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u tools::msg_writer() << "" << std::endl; tools::msg_writer() << "height: " << header.height << ", timestamp: " << header.timestamp << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" - << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl + << ", size: " << header.block_size << ", weight: " << header.block_weight << " (long term " << header.long_term_weight << "), transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -1640,14 +1641,14 @@ bool t_rpc_command_executor::print_bans() for (auto i = res.bans.begin(); i != res.bans.end(); ++i) { - tools::msg_writer() << epee::string_tools::get_ip_string_from_int32(i->ip) << " banned for " << i->seconds << " seconds"; + tools::msg_writer() << i->host << " banned for " << i->seconds << " seconds"; } return true; } -bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds) +bool t_rpc_command_executor::ban(const std::string &address, time_t seconds) { cryptonote::COMMAND_RPC_SETBANS::request req; cryptonote::COMMAND_RPC_SETBANS::response res; @@ -1655,11 +1656,8 @@ bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds) epee::json_rpc::error error_resp; cryptonote::COMMAND_RPC_SETBANS::ban ban; - if (!epee::string_tools::get_ip_int32_from_string(ban.ip, ip)) - { - tools::fail_msg_writer() << "Invalid IP"; - return true; - } + ban.host = address; + ban.ip = 0; ban.ban = true; ban.seconds = seconds; req.bans.push_back(ban); @@ -1683,7 +1681,7 @@ bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds) return true; } -bool t_rpc_command_executor::unban(const std::string &ip) +bool t_rpc_command_executor::unban(const std::string &address) { cryptonote::COMMAND_RPC_SETBANS::request req; cryptonote::COMMAND_RPC_SETBANS::response res; @@ -1691,11 +1689,8 @@ bool t_rpc_command_executor::unban(const std::string &ip) epee::json_rpc::error error_resp; cryptonote::COMMAND_RPC_SETBANS::ban ban; - if (!epee::string_tools::get_ip_int32_from_string(ban.ip, ip)) - { - tools::fail_msg_writer() << "Invalid IP"; - return true; - } + ban.host = address; + ban.ip = 0; ban.ban = false; ban.seconds = 0; req.bans.push_back(ban); @@ -1719,6 +1714,39 @@ bool t_rpc_command_executor::unban(const std::string &ip) return true; } +bool t_rpc_command_executor::banned(const std::string &address) +{ + cryptonote::COMMAND_RPC_BANNED::request req; + cryptonote::COMMAND_RPC_BANNED::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.address = address; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "banned", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_banned(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.banned) + tools::msg_writer() << address << " is banned for " << res.seconds << " seconds"; + else + tools::msg_writer() << address << " is not banned"; + + return true; +} + bool t_rpc_command_executor::flush_txpool(const std::string &txid) { cryptonote::COMMAND_RPC_FLUSH_TRANSACTION_POOL::request req; diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 3c2686b3f..61c25736d 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -43,6 +43,7 @@ #include "common/common_fwd.h" #include "common/rpc_client.h" #include "cryptonote_basic/cryptonote_basic.h" +#include "net/net_fwd.h" #include "rpc/core_rpc_server.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -61,6 +62,7 @@ public: uint32_t ip , uint16_t port , const boost::optional<tools::login>& user + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc = true , cryptonote::core_rpc_server* rpc_server = NULL ); @@ -135,9 +137,11 @@ public: bool print_bans(); - bool ban(const std::string &ip, time_t seconds); + bool ban(const std::string &address, time_t seconds); - bool unban(const std::string &ip); + bool unban(const std::string &address); + + bool banned(const std::string &address); bool flush_txpool(const std::string &txid); diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index 7bc2c324f..03c2b3e20 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -69,3 +69,25 @@ set_property(TARGET object_sizes PROPERTY OUTPUT_NAME "monero-utils-object-sizes") + +set(dns_checks_sources + dns_checks.cpp + ) + +monero_add_executable(dns_checks + ${dns_checks_sources} + ${dns_checks_private_headers}) + +target_link_libraries(dns_checks + LINK_PRIVATE + common + epee + version + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT}) + +set_property(TARGET dns_checks + PROPERTY + OUTPUT_NAME "monero-utils-dns-checks") + diff --git a/src/debug_utilities/dns_checks.cpp b/src/debug_utilities/dns_checks.cpp new file mode 100644 index 000000000..3c9daa769 --- /dev/null +++ b/src/debug_utilities/dns_checks.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string> +#include <vector> +#include <map> +#include <algorithm> +#include <boost/program_options.hpp> +#include "misc_log_ex.h" +#include "common/util.h" +#include "common/command_line.h" +#include "common/dns_utils.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "debugtools.dnschecks" + +namespace po = boost::program_options; + +enum lookup_t { LOOKUP_A, LOOKUP_TXT }; + +static std::vector<std::string> lookup(lookup_t type, const char *hostname) +{ + bool dnssec_available = false, dnssec_valid = false; + std::vector<std::string> res; + switch (type) + { + case LOOKUP_A: res = tools::DNSResolver::instance().get_ipv4(hostname, dnssec_available, dnssec_valid); break; + case LOOKUP_TXT: res = tools::DNSResolver::instance().get_txt_record(hostname, dnssec_available, dnssec_valid); break; + default: MERROR("Invalid lookup type: " << (int)type); return {}; + } + if (!dnssec_available) + { + MWARNING("No DNSSEC for " << hostname); + return {}; + } + if (!dnssec_valid) + { + MWARNING("Invalid DNSSEC check for " << hostname); + return {}; + } + MINFO(res.size() << " valid signed result(s) for " << hostname); + return res; +} + +static void lookup(lookup_t type, const std::vector<std::string> hostnames) +{ + std::vector<std::vector<std::string>> results; + for (const std::string &hostname: hostnames) + { + auto res = lookup(type, hostname.c_str()); + if (!res.empty()) + { + std::sort(res.begin(), res.end()); + results.push_back(res); + } + } + std::map<std::vector<std::string>, size_t> counter; + for (const auto &e: results) + counter[e]++; + size_t count = 0; + for (const auto &e: counter) + count = std::max(count, e.second); + if (results.size() > 1) + { + if (count < results.size()) + MERROR("Only " << count << "/" << results.size() << " records match"); + else + MINFO(count << "/" << results.size() << " records match"); + } +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + tools::on_startup(); + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure("", true); + mlog_set_categories("+" MONERO_DEFAULT_LOG_CATEGORY ":INFO"); + + lookup(LOOKUP_A, {"seeds.moneroseeds.se", "seeds.moneroseeds.ae.org", "seeds.moneroseeds.ch", "seeds.moneroseeds.li"}); + + lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se"}); + + lookup(LOOKUP_TXT, {"checkpoints.moneropulse.org", "checkpoints.moneropulse.net", "checkpoints.moneropulse.co", "checkpoints.moneropulse.se"}); + + // those are in the code, but don't seem to actually exist +#if 0 + lookup(LOOKUP_TXT, {"testpoints.moneropulse.org", "testpoints.moneropulse.net", "testpoints.moneropulse.co", "testpoints.moneropulse.se"); + + lookup(LOOKUP_TXT, {"stagenetpoints.moneropulse.org", "stagenetpoints.moneropulse.net", "stagenetpoints.moneropulse.co", "stagenetpoints.moneropulse.se"}); +#endif + + lookup(LOOKUP_TXT, {"segheights.moneropulse.org", "segheights.moneropulse.net", "segheights.moneropulse.co", "segheights.moneropulse.se"}); + + return 0; + CATCH_ENTRY_L0("main", 1); +} diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 200370564..eba633da8 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -90,6 +90,20 @@ namespace hw { AKout = keys.AKout; } + ABPkeys &ABPkeys::operator=(const ABPkeys& keys) { + if (&keys == this) + return *this; + Aout = keys.Aout; + Bout = keys.Bout; + is_subaddress = keys.is_subaddress; + is_change_address = keys.is_change_address; + additional_key = keys.additional_key; + index = keys.index; + Pout = keys.Pout; + AKout = keys.AKout; + return *this; + } + bool Keymap::find(const rct::key& P, ABPkeys& keys) const { size_t sz = ABP.size(); for (size_t i=0; i<sz; i++) { diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index d4d98ce4a..fe9028733 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -77,6 +77,7 @@ namespace hw { ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, bool is_subaddress, bool is_change_address, size_t index, const rct::key& P,const rct::key& AK); ABPkeys(const ABPkeys& keys) ; ABPkeys() {index=0;is_subaddress=false;is_subaddress=false;is_change_address=false;} + ABPkeys &operator=(const ABPkeys &keys); }; class Keymap { diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp index 359677064..91479521e 100644 --- a/src/lmdb/error.cpp +++ b/src/lmdb/error.cpp @@ -55,7 +55,7 @@ namespace { break; // map to nothing generic case MDB_PAGE_NOTFOUND: case MDB_CORRUPTED: - return std::errc::state_not_recoverable; + return std::errc::bad_address; case MDB_PANIC: case MDB_VERSION_MISMATCH: case MDB_INVALID: diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 653314b04..7d2599e9a 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -129,7 +129,7 @@ namespace Language case 1: *wptr++ = cp; break;
case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
- case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr += 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
default: throw std::runtime_error("Invalid UTF-8");
}
*wptr = 0;
diff --git a/src/net/error.h b/src/net/error.h index c8338f7e2..7c852dd20 100644 --- a/src/net/error.h +++ b/src/net/error.h @@ -42,7 +42,8 @@ namespace net invalid_i2p_address, invalid_port, //!< Outside of 0-65535 range invalid_tor_address,//!< Invalid base32 or length - unsupported_address //!< Type not supported by `get_network_address` + unsupported_address,//!< Type not supported by `get_network_address` + invalid_mask, //!< Outside of 0-32 range }; //! \return `std::error_category` for `net` namespace. diff --git a/src/net/parse.cpp b/src/net/parse.cpp index eaaadb67e..d93d7d352 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -58,4 +58,27 @@ namespace net return {epee::net_utils::ipv4_network_address{ip, port}}; return make_error_code(net::error::unsupported_address); } + + expect<epee::net_utils::ipv4_network_subnet> + get_ipv4_subnet_address(const boost::string_ref address, bool allow_implicit_32) + { + uint32_t mask = 32; + const boost::string_ref::size_type slash = address.find_first_of('/'); + if (slash != boost::string_ref::npos) + { + if (!epee::string_tools::get_xtype_from_string(mask, std::string{address.substr(slash + 1)})) + return make_error_code(net::error::invalid_mask); + if (mask > 32) + return make_error_code(net::error::invalid_mask); + } + else if (!allow_implicit_32) + return make_error_code(net::error::invalid_mask); + + std::uint32_t ip = 0; + boost::string_ref S(address.data(), slash != boost::string_ref::npos ? slash : address.size()); + if (!epee::string_tools::get_ip_int32_from_string(ip, std::string(S))) + return make_error_code(net::error::invalid_host); + + return {epee::net_utils::ipv4_network_subnet{ip, (uint8_t)mask}}; + } } diff --git a/src/net/parse.h b/src/net/parse.h index 5804c4128..9f0d66ea6 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -50,5 +50,18 @@ namespace net */ expect<epee::net_utils::network_address> get_network_address(boost::string_ref address, std::uint16_t default_port); + + /*! + Identifies an IPv4 subnet in CIDR notatioa and returns it as a generic + `network_address`. If the type is unsupported, it might be a hostname, + and `error() == net::error::kUnsupportedAddress` is returned. + + \param address An ipv4 address. + \param allow_implicit_32 whether to accept "raw" IPv4 addresses, with CIDR notation + + \return A tor or IPv4 address, else error. + */ + expect<epee::net_utils::ipv4_network_subnet> + get_ipv4_subnet_address(boost::string_ref address, bool allow_implicit_32 = false); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 42bb3b061..9ee5ce0de 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -248,7 +248,11 @@ namespace nodetool void change_max_in_public_peers(size_t count); virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME); virtual bool unblock_host(const epee::net_utils::network_address &address); - virtual std::map<std::string, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + virtual bool block_subnet(const epee::net_utils::ipv4_network_subnet &subnet, time_t seconds = P2P_IP_BLOCKTIME); + virtual bool unblock_subnet(const epee::net_utils::ipv4_network_subnet &subnet); + virtual bool is_host_blocked(const epee::net_utils::network_address &address, time_t *seconds) { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return !is_remote_host_allowed(address, seconds); } + virtual std::map<epee::net_utils::network_address, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_subnets; } virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); @@ -319,7 +323,7 @@ namespace nodetool virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); virtual bool add_host_fail(const epee::net_utils::network_address &address); //----------------- i_connection_filter -------------------------------------------------------- - virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address); + virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address, time_t *t = NULL); //----------------------------------------------------------------------------------------------- bool parse_peer_from_string(epee::net_utils::network_address& pe, const std::string& node_addr, uint16_t default_port = 0); bool handle_command_line( @@ -461,8 +465,9 @@ namespace nodetool std::map<epee::net_utils::network_address, time_t> m_conn_fails_cache; epee::critical_section m_conn_fails_cache_lock; - epee::critical_section m_blocked_hosts_lock; - std::map<std::string, time_t> m_blocked_hosts; + epee::critical_section m_blocked_hosts_lock; // for both hosts and subnets + std::map<epee::net_utils::network_address, time_t> m_blocked_hosts; + std::map<epee::net_utils::ipv4_network_subnet, time_t> m_blocked_subnets; epee::critical_section m_host_fails_score_lock; std::map<std::string, uint64_t> m_host_fails_score; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index be97edbe5..ae51a63b2 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -155,19 +155,55 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::is_remote_host_allowed(const epee::net_utils::network_address &address) + bool node_server<t_payload_net_handler>::is_remote_host_allowed(const epee::net_utils::network_address &address, time_t *t) { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); - auto it = m_blocked_hosts.find(address.host_str()); - if(it == m_blocked_hosts.end()) - return true; - if(time(nullptr) >= it->second) + + const time_t now = time(nullptr); + + // look in the hosts list + auto it = m_blocked_hosts.find(address); + if (it != m_blocked_hosts.end()) { - m_blocked_hosts.erase(it); - MCLOG_CYAN(el::Level::Info, "global", "Host " << address.host_str() << " unblocked."); - return true; + if (now >= it->second) + { + m_blocked_hosts.erase(it); + MCLOG_CYAN(el::Level::Info, "global", "Host " << address.host_str() << " unblocked."); + it = m_blocked_hosts.end(); + } + else + { + if (t) + *t = it->second - now; + return false; + } } - return false; + + // manually loop in subnets + if (address.get_type_id() == epee::net_utils::address_type::ipv4) + { + auto ipv4_address = address.template as<epee::net_utils::ipv4_network_address>(); + std::map<epee::net_utils::ipv4_network_subnet, time_t>::iterator it; + for (it = m_blocked_subnets.begin(); it != m_blocked_subnets.end(); ) + { + if (now >= it->second) + { + it = m_blocked_subnets.erase(it); + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << it->first.host_str() << " unblocked."); + continue; + } + if (it->first.matches(ipv4_address)) + { + if (t) + *t = it->second - now; + return false; + } + ++it; + } + } + + // not found in hosts or subnets, allowed + return true; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -184,7 +220,7 @@ namespace nodetool limit = std::numeric_limits<time_t>::max(); else limit = now + seconds; - m_blocked_hosts[addr.host_str()] = limit; + m_blocked_hosts[addr] = limit; // drop any connection to that address. This should only have to look into // the zone related to the connection, but really make sure everything is @@ -214,7 +250,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::unblock_host(const epee::net_utils::network_address &address) { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); - auto i = m_blocked_hosts.find(address.host_str()); + auto i = m_blocked_hosts.find(address); if (i == m_blocked_hosts.end()) return false; m_blocked_hosts.erase(i); @@ -223,6 +259,58 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::block_subnet(const epee::net_utils::ipv4_network_subnet &subnet, time_t seconds) + { + const time_t now = time(nullptr); + + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); + time_t limit; + if (now > std::numeric_limits<time_t>::max() - seconds) + limit = std::numeric_limits<time_t>::max(); + else + limit = now + seconds; + m_blocked_subnets[subnet] = limit; + + // drop any connection to that subnet. This should only have to look into + // the zone related to the connection, but really make sure everything is + // swept ... + std::vector<boost::uuids::uuid> conns; + for(auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (cntxt.m_remote_address.get_type_id() != epee::net_utils::ipv4_network_address::get_type_id()) + return true; + auto ipv4_address = cntxt.m_remote_address.template as<epee::net_utils::ipv4_network_address>(); + if (subnet.matches(ipv4_address)) + { + conns.push_back(cntxt.m_connection_id); + } + return true; + }); + for (const auto &c: conns) + zone.second.m_net_server.get_config_object().close(c); + + conns.clear(); + } + + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked."); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::unblock_subnet(const epee::net_utils::ipv4_network_subnet &subnet) + { + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); + auto i = m_blocked_subnets.find(subnet); + if (i == m_blocked_subnets.end()) + return false; + m_blocked_subnets.erase(i); + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " unblocked."); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::add_host_fail(const epee::net_utils::network_address &address) { if(!address.is_blockable()) @@ -944,7 +1032,10 @@ namespace nodetool } if(!context.m_is_income) m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port); - m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false); + if (!m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false)) + { + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); + } }); if(!r) @@ -1090,6 +1181,7 @@ namespace nodetool LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str() /*<< ", try " << try_count*/); + zone.m_net_server.get_config_object().close(con->m_connection_id); return false; } @@ -1149,7 +1241,7 @@ namespace nodetool bool is_priority = is_priority_node(na); LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str()); - + zone.m_net_server.get_config_object().close(con->m_connection_id); return false; } @@ -1226,19 +1318,53 @@ namespace nodetool size_t random_index; const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second; + // build a set of all the /16 we're connected to, and prefer a peer that's not in that set + std::set<uint32_t> classB; + if (&zone == &m_network_zones.at(epee::net_utils::zone::public_)) // at returns reference, not copy + { + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + + const epee::net_utils::network_address na = cntxt.m_remote_address; + const uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip(); + classB.insert(actual_ip & 0x0000ffff); + } + return true; + }); + } + std::deque<size_t> filtered; const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max(); - size_t idx = 0; - zone.m_peerlist.foreach (use_white_list, [&filtered, &idx, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ - if (filtered.size() >= limit) - return false; - if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) - filtered.push_back(idx); - else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) - filtered.push_front(idx); - ++idx; - return true; - }); + size_t idx = 0, skipped = 0; + for (int step = 0; step < 2; ++step) + { + bool skip_duplicate_class_B = step == 0; + zone.m_peerlist.foreach (use_white_list, [&classB, &filtered, &idx, &skipped, skip_duplicate_class_B, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ + if (filtered.size() >= limit) + return false; + bool skip = false; + if (skip_duplicate_class_B && pe.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + const epee::net_utils::network_address na = pe.adr; + uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip(); + skip = classB.find(actual_ip & 0x0000ffff) != classB.end(); + } + if (skip) + ++skipped; + else if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) + filtered.push_back(idx); + else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) + filtered.push_front(idx); + ++idx; + return true; + }); + if (skipped == 0 || !filtered.empty()) + break; + if (skipped) + MGINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); + } if (filtered.empty()) { MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe); @@ -1955,7 +2081,7 @@ namespace nodetool const epee::net_utils::zone zone_type = context.m_remote_address.get_zone(); network_zone& zone = m_network_zones.at(zone_type); - zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); m_payload_handler.get_payload_sync_data(rsp.payload_data); /* Tor/I2P nodes receiving connections via forwarding (from tor/i2p daemon) @@ -2058,7 +2184,7 @@ namespace nodetool }); //fill response - zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); get_local_node_data(rsp.node_data, zone); m_payload_handler.get_payload_sync_data(rsp.payload_data); LOG_DEBUG_CC(context, "COMMAND_HANDSHAKE"); diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 26451b333..34d151f5f 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -56,7 +56,8 @@ namespace nodetool virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0; virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds = 0)=0; virtual bool unblock_host(const epee::net_utils::network_address &address)=0; - virtual std::map<std::string, time_t> get_blocked_hosts()=0; + virtual std::map<epee::net_utils::network_address, time_t> get_blocked_hosts()=0; + virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets()=0; virtual bool add_host_fail(const epee::net_utils::network_address &address)=0; virtual void add_used_stripe_peer(const t_connection_context &context)=0; virtual void remove_used_stripe_peer(const t_connection_context &context)=0; @@ -112,9 +113,13 @@ namespace nodetool { return true; } - virtual std::map<std::string, time_t> get_blocked_hosts() + virtual std::map<epee::net_utils::network_address, time_t> get_blocked_hosts() { - return std::map<std::string, time_t>(); + return std::map<epee::net_utils::network_address, time_t>(); + } + virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets() + { + return std::map<epee::net_utils::ipv4_network_subnet, time_t>(); } virtual bool add_host_fail(const epee::net_utils::network_address &address) { diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index 52814af94..883997fd6 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -102,7 +102,7 @@ namespace nodetool size_t get_white_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_white.size();} size_t get_gray_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_gray.size();} bool merge_peerlist(const std::vector<peerlist_entry>& outer_bs); - bool get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); + bool get_peerlist_head(std::vector<peerlist_entry>& bs_head, bool anonymize, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); void get_peerlist(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white); void get_peerlist(peerlist_types& peers); bool get_white_peer_by_index(peerlist_entry& p, size_t i); @@ -263,23 +263,40 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth) + bool peerlist_manager::get_peerlist_head(std::vector<peerlist_entry>& bs_head, bool anonymize, uint32_t depth) { - CRITICAL_REGION_LOCAL(m_peerlist_lock); peers_indexed::index<by_time>::type& by_time_index=m_peers_white.get<by_time>(); uint32_t cnt = 0; - bs_head.reserve(depth); + + // picks a random set of peers within the first 120%, rather than a set of the first 100%. + // The intent is that if someone asks twice, they can't easily tell: + // - this address was not in the first list, but is in the second, so the only way this can be + // is if its last_seen was recently reset, so this means the target node recently had a new + // connection to that address + // - this address was in the first list, and not in the second, which means either the address + // was moved to the gray list (if it's not accessibe, which the attacker can check if + // the address accepts incoming connections) or it was the oldest to still fit in the 250 items, + // so its last_seen is old. + const uint32_t pick_depth = anonymize ? depth + depth / 5 : depth; + bs_head.reserve(pick_depth); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) { - if(!vl.last_seen) - continue; - - if(cnt++ >= depth) + if(cnt++ >= pick_depth) break; bs_head.push_back(vl); } + + if (anonymize) + { + std::random_shuffle(bs_head.begin(), bs_head.end()); + if (bs_head.size() > depth) + bs_head.resize(depth); + for (auto &e: bs_head) + e.last_seen = 0; + } + return true; } //-------------------------------------------------------------------------------------------------- @@ -327,8 +344,14 @@ namespace nodetool trim_white_peerlist(); }else { - //update record in white list - m_peers_white.replace(by_addr_it_wt, ple); + //update record in white list + peerlist_entry new_ple = ple; + if (by_addr_it_wt->pruning_seed && ple.pruning_seed == 0) // guard against older nodes not passing pruning info around + new_ple.pruning_seed = by_addr_it_wt->pruning_seed; + if (by_addr_it_wt->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around + new_ple.rpc_port = by_addr_it_wt->rpc_port; + new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + m_peers_white.replace(by_addr_it_wt, new_ple); } //remove from gray list, if need auto by_addr_it_gr = m_peers_gray.get<by_addr>().find(ple.adr); @@ -362,8 +385,14 @@ namespace nodetool trim_gray_peerlist(); }else { - //update record in white list - m_peers_gray.replace(by_addr_it_gr, ple); + //update record in gray list + peerlist_entry new_ple = ple; + if (by_addr_it_gr->pruning_seed && ple.pruning_seed == 0) // guard against older nodes not passing pruning info around + new_ple.pruning_seed = by_addr_it_gr->pruning_seed; + if (by_addr_it_gr->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around + new_ple.rpc_port = by_addr_it_gr->rpc_port; + new_ple.last_seen = by_addr_it_gr->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + m_peers_gray.replace(by_addr_it_gr, new_ple); } return true; CATCH_ENTRY_L0("peerlist_manager::append_with_peer_gray()", false); diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 40ef2ebcd..32f30adca 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -134,10 +134,11 @@ namespace boost a & port; a & length; - if (length > net::tor_address::buffer_size()) + const size_t buffer_size = net::tor_address::buffer_size(); + if (length > buffer_size) MONERO_THROW(net::error::invalid_tor_address, "Tor address too long"); - char host[net::tor_address::buffer_size()] = {0}; + char host[buffer_size] = {0}; a.load_binary(host, length); host[sizeof(host) - 1] = 0; @@ -155,10 +156,11 @@ namespace boost a & port; a & length; - if (length > net::i2p_address::buffer_size()) + const size_t buffer_size = net::i2p_address::buffer_size(); + if (length > buffer_size) MONERO_THROW(net::error::invalid_i2p_address, "i2p address too long"); - char host[net::i2p_address::buffer_size()] = {0}; + char host[buffer_size] = {0}; a.load_binary(host, length); host[sizeof(host) - 1] = 0; diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index 59c6099d5..85774fcd5 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -81,7 +81,8 @@ namespace nodetool BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(adr) KV_SERIALIZE(id) - KV_SERIALIZE(last_seen) + if (!is_store || this_ref.last_seen != 0) + KV_SERIALIZE_OPT(last_seen, (int64_t)0) KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) END_KV_SERIALIZE_MAP() @@ -132,7 +133,7 @@ namespace nodetool ss << pe.id << "\t" << pe.adr.str() << " \trpc port " << (pe.rpc_port > 0 ? std::to_string(pe.rpc_port) : "-") << " \tpruning seed " << pe.pruning_seed - << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) + << " \tlast_seen: " << (pe.last_seen == 0 ? std::string("never") : epee::misc_utils::get_time_interval_string(now_time - pe.last_seen)) << std::endl; } return ss.str(); diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 6f77fed34..f69b4a12c 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -447,7 +447,6 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str { CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); - bool HiGi = cache != NULL; STEP = STEP ? STEP : 192; MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index f01e683cb..2c4e5fc3b 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -190,7 +190,6 @@ namespace rct { int byte, i, j; for (j = 0; j < 8; j++) { byte = 0; - i = 8 * j; for (i = 7; i > -1; i--) { byte = byte * 2 + amountb2[8 * j + i]; } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index e5413f1dc..f8729b872 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -252,7 +252,7 @@ namespace rct { { FIELD(type) if (type == RCTTypeNull) - return true; + return ar.stream().good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; VARINT_FIELD(txnFee) @@ -312,7 +312,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - return true; + return ar.stream().good(); } }; struct rctSigPrunable { @@ -325,7 +325,7 @@ namespace rct { bool serialize_rctsig_prunable(Archive<W> &ar, uint8_t type, size_t inputs, size_t outputs, size_t mixin) { if (type == RCTTypeNull) - return true; + return ar.stream().good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) @@ -429,7 +429,7 @@ namespace rct { } ar.end_array(); } - return true; + return ar.stream().good(); } }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 97e9b482e..82da06bc5 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -59,6 +59,8 @@ using namespace epee; #define MAX_RESTRICTED_FAKE_OUTS_COUNT 40 #define MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT 5000 +#define OUTPUT_HISTOGRAM_RECENT_CUTOFF_RESTRICTION (3 * 86400) // 3 days max, the wallet requests 1.8 days + namespace { void add_reason(std::string &reasons, const char *reason) @@ -75,9 +77,9 @@ namespace void store_difficulty(cryptonote::difficulty_type difficulty, uint64_t &sdiff, std::string &swdiff, uint64_t &stop64) { - sdiff = (difficulty << 64 >> 64).convert_to<uint64_t>(); + sdiff = (difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); swdiff = cryptonote::hex(difficulty); - stop64 = (difficulty >> 64).convert_to<uint64_t>(); + stop64 = ((difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); } } @@ -90,15 +92,9 @@ namespace cryptonote command_line::add_arg(desc, arg_rpc_bind_port); command_line::add_arg(desc, arg_rpc_restricted_bind_port); command_line::add_arg(desc, arg_restricted_rpc); - command_line::add_arg(desc, arg_rpc_ssl); - command_line::add_arg(desc, arg_rpc_ssl_private_key); - command_line::add_arg(desc, arg_rpc_ssl_certificate); - command_line::add_arg(desc, arg_rpc_ssl_ca_certificates); - command_line::add_arg(desc, arg_rpc_ssl_allowed_fingerprints); - command_line::add_arg(desc, arg_rpc_ssl_allow_any_cert); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); - cryptonote::rpc_args::init_options(desc); + cryptonote::rpc_args::init_options(desc, true); } //------------------------------------------------------------------------------------------------------------------------------ core_rpc_server::core_rpc_server( @@ -118,7 +114,7 @@ namespace cryptonote m_restricted = restricted; m_net_server.set_threads_prefix("RPC"); - auto rpc_config = cryptonote::rpc_args::process(vm); + auto rpc_config = cryptonote::rpc_args::process(vm, true); if (!rpc_config) return false; @@ -151,46 +147,9 @@ namespace cryptonote if (rpc_config->login) http_login.emplace(std::move(rpc_config->login->username), std::move(rpc_config->login->password).password()); - epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_autodetect; - if (command_line::get_arg(vm, arg_rpc_ssl_allow_any_cert)) - ssl_options.verification = epee::net_utils::ssl_verification_t::none; - else - { - std::string ssl_ca_path = command_line::get_arg(vm, arg_rpc_ssl_ca_certificates); - const std::vector<std::string> ssl_allowed_fingerprint_strings = command_line::get_arg(vm, arg_rpc_ssl_allowed_fingerprints); - std::vector<std::vector<uint8_t>> ssl_allowed_fingerprints{ ssl_allowed_fingerprint_strings.size() }; - std::transform(ssl_allowed_fingerprint_strings.begin(), ssl_allowed_fingerprint_strings.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); - for (const auto &fpr: ssl_allowed_fingerprints) - { - if (fpr.size() != SSL_FINGERPRINT_SIZE) - { - MERROR("SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); - return false; - } - } - - if (!ssl_ca_path.empty() || !ssl_allowed_fingerprints.empty()) - ssl_options = epee::net_utils::ssl_options_t{std::move(ssl_allowed_fingerprints), std::move(ssl_ca_path)}; - } - - ssl_options.auth = epee::net_utils::ssl_authentication_t{ - command_line::get_arg(vm, arg_rpc_ssl_private_key), command_line::get_arg(vm, arg_rpc_ssl_certificate) - }; - - // user specified CA file or fingeprints implies enabled SSL by default - if (ssl_options.verification != epee::net_utils::ssl_verification_t::user_certificates || !command_line::is_arg_defaulted(vm, arg_rpc_ssl)) - { - const std::string ssl = command_line::get_arg(vm, arg_rpc_ssl); - if (!epee::net_utils::ssl_support_from_string(ssl_options.support, ssl)) - { - MFATAL("Invalid RPC SSL support: " << ssl); - return false; - } - } - auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; return epee::http_server_impl_base<core_rpc_server, connection_context>::init( - rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(ssl_options) + rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -861,6 +820,7 @@ namespace cryptonote res.sanity_check_failed = true; return true; } + res.sanity_check_failed = false; cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -948,16 +908,13 @@ namespace cryptonote return true; } - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - cryptonote::miner &miner= m_core.get_miner(); if (miner.is_mining()) { res.status = "Already mining"; return true; } - if(!miner.start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + if(!miner.start(info.address, static_cast<size_t>(req.threads_count), req.do_background_mining, req.ignore_battery)) { res.status = "Failed, mining not started"; LOG_PRINT_L0(res.status); @@ -1519,7 +1476,7 @@ namespace cryptonote ok = ok && getheight_res.status == CORE_RPC_STATUS_OK; m_should_use_bootstrap_daemon = ok && top_height + 10 < getheight_res.height; - MINFO((m_should_use_bootstrap_daemon ? "Using" : "Not using") << " the bootstrap daemon (our height: " << top_height << ", bootstrap daemon's height: " << getheight_res.height << ")"); + MINFO((m_should_use_bootstrap_daemon ? "Using" : "Not using") << " the bootstrap daemon (our height: " << top_height << ", bootstrap daemon's height: " << (ok ? getheight_res.height : 0) << ")"); } if (!m_should_use_bootstrap_daemon) return false; @@ -1815,20 +1772,60 @@ namespace cryptonote PERF_TIMER(on_get_bans); auto now = time(nullptr); - std::map<std::string, time_t> blocked_hosts = m_p2p.get_blocked_hosts(); - for (std::map<std::string, time_t>::const_iterator i = blocked_hosts.begin(); i != blocked_hosts.end(); ++i) + std::map<epee::net_utils::network_address, time_t> blocked_hosts = m_p2p.get_blocked_hosts(); + for (std::map<epee::net_utils::network_address, time_t>::const_iterator i = blocked_hosts.begin(); i != blocked_hosts.end(); ++i) { if (i->second > now) { COMMAND_RPC_GETBANS::ban b; - b.host = i->first; + b.host = i->first.host_str(); b.ip = 0; uint32_t ip; - if (epee::string_tools::get_ip_int32_from_string(ip, i->first)) + if (epee::string_tools::get_ip_int32_from_string(ip, b.host)) b.ip = ip; b.seconds = i->second - now; res.bans.push_back(b); } } + std::map<epee::net_utils::ipv4_network_subnet, time_t> blocked_subnets = m_p2p.get_blocked_subnets(); + for (std::map<epee::net_utils::ipv4_network_subnet, time_t>::const_iterator i = blocked_subnets.begin(); i != blocked_subnets.end(); ++i) + { + if (i->second > now) { + COMMAND_RPC_GETBANS::ban b; + b.host = i->first.host_str(); + b.ip = 0; + b.seconds = i->second - now; + res.bans.push_back(b); + } + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_banned(const COMMAND_RPC_BANNED::request& req, COMMAND_RPC_BANNED::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + PERF_TIMER(on_banned); + + auto na_parsed = net::get_network_address(req.address, 0); + if (!na_parsed) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Unsupported host type"; + return false; + } + epee::net_utils::network_address na = std::move(*na_parsed); + + time_t seconds; + if (m_p2p.is_host_blocked(na, &seconds)) + { + res.banned = true; + res.seconds = seconds; + } + else + { + res.banned = false; + res.seconds = 0; + } res.status = CORE_RPC_STATUS_OK; return true; @@ -1841,13 +1838,29 @@ namespace cryptonote for (auto i = req.bans.begin(); i != req.bans.end(); ++i) { epee::net_utils::network_address na; + + // try subnet first + if (!i->host.empty()) + { + auto ns_parsed = net::get_ipv4_subnet_address(i->host); + if (ns_parsed) + { + if (i->ban) + m_p2p.block_subnet(*ns_parsed, i->seconds); + else + m_p2p.unblock_subnet(*ns_parsed); + continue; + } + } + + // then host if (!i->host.empty()) { auto na_parsed = net::get_network_address(i->host, 0); if (!na_parsed) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; - error_resp.message = "Unsupported host type"; + error_resp.message = "Unsupported host/subnet type"; return false; } na = std::move(*na_parsed); @@ -1928,6 +1941,13 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_HISTOGRAM>(invoke_http_mode::JON_RPC, "get_output_histogram", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + if (restricted && req.recent_cutoff > 0 && req.recent_cutoff < (uint64_t)time(NULL) - OUTPUT_HISTOGRAM_RECENT_CUTOFF_RESTRICTION) + { + res.status = "Recent cutoff is too old"; + return true; + } + std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> histogram; try { @@ -2107,6 +2127,13 @@ namespace cryptonote bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res, const connection_context *ctx) { PERF_TIMER(on_update); + + if (m_core.offline()) + { + res.status = "Daemon is running offline"; + return true; + } + static const char software[] = "monero"; #ifdef BUILD_TAG static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG); @@ -2317,7 +2344,7 @@ namespace cryptonote const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -2360,7 +2387,7 @@ namespace cryptonote const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { res.status = "Failed to get output distribution"; @@ -2432,40 +2459,6 @@ namespace cryptonote , false }; - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl = { - "rpc-ssl" - , "Enable SSL on RPC connections: enabled|disabled|autodetect" - , "autodetect" - }; - - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_private_key = { - "rpc-ssl-private-key" - , "Path to a PEM format private key" - , "" - }; - - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_certificate = { - "rpc-ssl-certificate" - , "Path to a PEM format certificate" - , "" - }; - - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_ca_certificates = { - "rpc-ssl-ca-certificates" - , "Path to file containing concatenated PEM format certificate(s) to replace system CA(s)." - }; - - const command_line::arg_descriptor<std::vector<std::string>> core_rpc_server::arg_rpc_ssl_allowed_fingerprints = { - "rpc-ssl-allowed-fingerprints" - , "List of certificate fingerprints to allow" - }; - - const command_line::arg_descriptor<bool> core_rpc_server::arg_rpc_ssl_allow_any_cert = { - "rpc-ssl-allow-any-cert" - , "Allow any peer certificate" - , false - }; - const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_address = { "bootstrap-daemon-address" , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index e4683bbe2..266661fb0 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -154,6 +154,7 @@ namespace cryptonote MAP_JON_RPC_WE("hard_fork_info", on_hard_fork_info, COMMAND_RPC_HARD_FORK_INFO) MAP_JON_RPC_WE_IF("set_bans", on_set_bans, COMMAND_RPC_SETBANS, !m_restricted) MAP_JON_RPC_WE_IF("get_bans", on_get_bans, COMMAND_RPC_GETBANS, !m_restricted) + MAP_JON_RPC_WE_IF("banned", on_banned, COMMAND_RPC_BANNED, !m_restricted) MAP_JON_RPC_WE_IF("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL, !m_restricted) MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) @@ -220,6 +221,7 @@ namespace cryptonote bool on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_banned(const COMMAND_RPC_BANNED::request& req, COMMAND_RPC_BANNED::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index cfe4bbf23..a78faf5aa 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -84,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 6 +#define CORE_RPC_VERSION_MINOR 7 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -1876,6 +1876,33 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_BANNED + { + struct request_t + { + std::string address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string status; + bool banned; + uint32_t seconds; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(banned) + KV_SERIALIZE(seconds) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_FLUSH_TRANSACTION_POOL { struct request_t diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 7c8953930..612b2cab6 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -408,10 +408,7 @@ namespace rpc return; } - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - if(!m_core.get_miner().start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + if(!m_core.get_miner().start(info.address, static_cast<size_t>(req.threads_count), req.do_background_mining, req.ignore_battery)) { res.error_details = "Failed, mining not started"; LOG_PRINT_L0(res.error_details); @@ -437,7 +434,7 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); res.info.wide_difficulty = chain.get_difficulty_for_next_block(); - res.info.difficulty = (res.info.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); + res.info.difficulty = (res.info.wide_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); res.info.target = chain.get_difficulty_target(); @@ -459,7 +456,7 @@ namespace rpc res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; res.info.wide_cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); @@ -778,7 +775,7 @@ namespace rpc const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (std::uint64_t amount : req.amounts) { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { res.distributions.clear(); @@ -829,7 +826,7 @@ namespace rpc } header.wide_difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); - header.difficulty = (header.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); + header.difficulty = (header.wide_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); return true; } diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index f2be94f51..4479bd1f1 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -33,28 +33,95 @@ #include <boost/bind.hpp> #include "common/command_line.h" #include "common/i18n.h" +#include "hex.h" namespace cryptonote { + namespace + { + boost::optional<epee::net_utils::ssl_options_t> do_process_ssl(const boost::program_options::variables_map& vm, const rpc_args::descriptors& arg, const bool any_cert_option) + { + bool ssl_required = false; + epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; + if (any_cert_option && command_line::get_arg(vm, arg.rpc_ssl_allow_any_cert)) + ssl_options.verification = epee::net_utils::ssl_verification_t::none; + else + { + std::string ssl_ca_file = command_line::get_arg(vm, arg.rpc_ssl_ca_certificates); + const std::vector<std::string> ssl_allowed_fingerprints = command_line::get_arg(vm, arg.rpc_ssl_allowed_fingerprints); + + std::vector<std::vector<uint8_t>> allowed_fingerprints{ ssl_allowed_fingerprints.size() }; + std::transform(ssl_allowed_fingerprints.begin(), ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); + for (const auto &fpr: allowed_fingerprints) + { + if (fpr.size() != SSL_FINGERPRINT_SIZE) + { + MERROR("SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); + return boost::none; + } + } + + if (!allowed_fingerprints.empty() || !ssl_ca_file.empty()) + { + ssl_required = true; + ssl_options = epee::net_utils::ssl_options_t{ + std::move(allowed_fingerprints), std::move(ssl_ca_file) + }; + + if (command_line::get_arg(vm, arg.rpc_ssl_allow_chained)) + ssl_options.verification = epee::net_utils::ssl_verification_t::user_ca; + } + } + + // user specified CA file or fingeprints implies enabled SSL by default + if (!ssl_required && !epee::net_utils::ssl_support_from_string(ssl_options.support, command_line::get_arg(vm, arg.rpc_ssl))) + { + MERROR("Invalid argument for " << std::string(arg.rpc_ssl.name)); + return boost::none; + } + + ssl_options.auth = epee::net_utils::ssl_authentication_t{ + command_line::get_arg(vm, arg.rpc_ssl_private_key), command_line::get_arg(vm, arg.rpc_ssl_certificate) + }; + + return {std::move(ssl_options)}; + } + } // anonymous + rpc_args::descriptors::descriptors() : rpc_bind_ip({"rpc-bind-ip", rpc_args::tr("Specify IP to bind RPC server"), "127.0.0.1"}) , rpc_login({"rpc-login", rpc_args::tr("Specify username[:password] required for RPC server"), "", true}) , confirm_external_bind({"confirm-external-bind", rpc_args::tr("Confirm rpc-bind-ip value is NOT a loopback (local) IP")}) , rpc_access_control_origins({"rpc-access-control-origins", rpc_args::tr("Specify a comma separated list of origins to allow cross origin resource sharing"), ""}) + , rpc_ssl({"rpc-ssl", rpc_args::tr("Enable SSL on RPC connections: enabled|disabled|autodetect"), "autodetect"}) + , rpc_ssl_private_key({"rpc-ssl-private-key", rpc_args::tr("Path to a PEM format private key"), ""}) + , rpc_ssl_certificate({"rpc-ssl-certificate", rpc_args::tr("Path to a PEM format certificate"), ""}) + , rpc_ssl_ca_certificates({"rpc-ssl-ca-certificates", rpc_args::tr("Path to file containing concatenated PEM format certificate(s) to replace system CA(s)."), ""}) + , rpc_ssl_allowed_fingerprints({"rpc-ssl-allowed-fingerprints", rpc_args::tr("List of certificate fingerprints to allow")}) + , rpc_ssl_allow_chained({"rpc-ssl-allow-chained", rpc_args::tr("Allow user (via --rpc-ssl-certificates) chain certificates"), false}) + , rpc_ssl_allow_any_cert({"rpc-ssl-allow-any-cert", rpc_args::tr("Allow any peer certificate"), false}) {} const char* rpc_args::tr(const char* str) { return i18n_translate(str, "cryptonote::rpc_args"); } - void rpc_args::init_options(boost::program_options::options_description& desc) + void rpc_args::init_options(boost::program_options::options_description& desc, const bool any_cert_option) { const descriptors arg{}; command_line::add_arg(desc, arg.rpc_bind_ip); command_line::add_arg(desc, arg.rpc_login); command_line::add_arg(desc, arg.confirm_external_bind); command_line::add_arg(desc, arg.rpc_access_control_origins); + command_line::add_arg(desc, arg.rpc_ssl); + command_line::add_arg(desc, arg.rpc_ssl_private_key); + command_line::add_arg(desc, arg.rpc_ssl_certificate); + command_line::add_arg(desc, arg.rpc_ssl_ca_certificates); + command_line::add_arg(desc, arg.rpc_ssl_allowed_fingerprints); + command_line::add_arg(desc, arg.rpc_ssl_allow_chained); + if (any_cert_option) + command_line::add_arg(desc, arg.rpc_ssl_allow_any_cert); } - boost::optional<rpc_args> rpc_args::process(const boost::program_options::variables_map& vm) + boost::optional<rpc_args> rpc_args::process(const boost::program_options::variables_map& vm, const bool any_cert_option) { const descriptors arg{}; rpc_args config{}; @@ -118,6 +185,17 @@ namespace cryptonote config.access_control_origins = std::move(access_control_origins); } + auto ssl_options = do_process_ssl(vm, arg, any_cert_option); + if (!ssl_options) + return boost::none; + config.ssl_options = std::move(*ssl_options); + return {std::move(config)}; } + + boost::optional<epee::net_utils::ssl_options_t> rpc_args::process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option) + { + const descriptors arg{}; + return do_process_ssl(vm, arg, any_cert_option); + } } diff --git a/src/rpc/rpc_args.h b/src/rpc/rpc_args.h index 216ba3712..619f02b42 100644 --- a/src/rpc/rpc_args.h +++ b/src/rpc/rpc_args.h @@ -35,6 +35,7 @@ #include "common/command_line.h" #include "common/password.h" +#include "net/net_ssl.h" namespace cryptonote { @@ -54,16 +55,29 @@ namespace cryptonote const command_line::arg_descriptor<std::string> rpc_login; const command_line::arg_descriptor<bool> confirm_external_bind; const command_line::arg_descriptor<std::string> rpc_access_control_origins; + const command_line::arg_descriptor<std::string> rpc_ssl; + const command_line::arg_descriptor<std::string> rpc_ssl_private_key; + const command_line::arg_descriptor<std::string> rpc_ssl_certificate; + const command_line::arg_descriptor<std::string> rpc_ssl_ca_certificates; + const command_line::arg_descriptor<std::vector<std::string>> rpc_ssl_allowed_fingerprints; + const command_line::arg_descriptor<bool> rpc_ssl_allow_chained; + const command_line::arg_descriptor<bool> rpc_ssl_allow_any_cert; }; + // `allow_any_cert` bool toggles `--rpc-ssl-allow-any-cert` configuration + static const char* tr(const char* str); - static void init_options(boost::program_options::options_description& desc); + static void init_options(boost::program_options::options_description& desc, const bool any_cert_option = false); //! \return Arguments specified by user, or `boost::none` if error - static boost::optional<rpc_args> process(const boost::program_options::variables_map& vm); + static boost::optional<rpc_args> process(const boost::program_options::variables_map& vm, const bool any_cert_option = false); + + //! \return SSL arguments specified by user, or `boost::none` if error + static boost::optional<epee::net_utils::ssl_options_t> process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option = false); std::string bind_ip; std::vector<std::string> access_control_origins; boost::optional<tools::login> login; // currently `boost::none` if unspecified by user + epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; }; } diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp index e0a81c70f..af5cb98a3 100644 --- a/src/rpc/rpc_handler.cpp +++ b/src/rpc/rpc_handler.cpp @@ -26,26 +26,49 @@ namespace rpc } boost::optional<output_distribution_data> - RpcHandler::get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, bool cumulative) + RpcHandler::get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height) { static struct D { boost::mutex mutex; std::vector<std::uint64_t> cached_distribution; std::uint64_t cached_from, cached_to, cached_start_height, cached_base; + crypto::hash cached_m10_hash; + crypto::hash cached_top_hash; bool cached; - D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} + D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached_m10_hash(crypto::null_hash), cached_top_hash(crypto::null_hash), cached(false) {} } d; const boost::unique_lock<boost::mutex> lock(d.mutex); - if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height) + crypto::hash top_hash = crypto::null_hash; + if (d.cached_to < blockchain_height) + top_hash = get_hash(d.cached_to); + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height && d.cached_top_hash == top_hash) return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base); std::vector<std::uint64_t> distribution; std::uint64_t start_height, base; // see if we can extend the cache - a common case - if (d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to) + bool can_extend = d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to && top_hash == d.cached_top_hash; + if (!can_extend) + { + // we kept track of the hash 10 blocks below, if it exists, so if it matches, + // we can still pop the last 10 cached slots and try again + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to - d.cached_from >= 10 && to_height > d.cached_to - 10) + { + crypto::hash hash10 = get_hash(d.cached_to - 10); + if (hash10 == d.cached_m10_hash) + { + d.cached_to -= 10; + d.cached_top_hash = hash10; + d.cached_m10_hash = crypto::null_hash; + d.cached_distribution.resize(d.cached_distribution.size() - 10); + can_extend = true; + } + } + } + if (can_extend) { std::vector<std::uint64_t> new_distribution; if (!f(amount, d.cached_to + 1, to_height, start_height, new_distribution, base)) @@ -74,6 +97,8 @@ namespace rpc { d.cached_from = from_height; d.cached_to = to_height; + d.cached_top_hash = get_hash(d.cached_to); + d.cached_m10_hash = d.cached_to >= 10 ? get_hash(d.cached_to - 10) : crypto::null_hash; d.cached_distribution = distribution; d.cached_start_height = start_height; d.cached_base = base; diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 2439eaa58..b81983d28 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -32,6 +32,7 @@ #include <cstdint> #include <string> #include <vector> +#include "crypto/hash.h" namespace cryptonote { @@ -56,7 +57,7 @@ class RpcHandler virtual std::string handle(const std::string& request) = 0; static boost::optional<output_distribution_data> - get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, bool cumulative); + get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height); }; diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index a0e4eff9d..9f60cf311 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -146,7 +146,8 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> void serialize_uvarint(T &v) { typedef std::istreambuf_iterator<char> it; - tools::read_varint(it(stream_), it(), v); // XXX handle failure + if (tools::read_varint(it(stream_), it(), v) < 0) + stream_.setstate(std::ios_base::failbit); } void begin_array(size_t &s) diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h index e32e24b78..c551095f9 100644 --- a/src/serialization/difficulty_type.h +++ b/src/serialization/difficulty_type.h @@ -54,8 +54,8 @@ inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff) { if (!ar.stream().good()) return false; - const uint64_t hi = (diff >> 64).convert_to<uint64_t>(); - const uint64_t lo = (diff << 64 >> 64).convert_to<uint64_t>(); + const uint64_t hi = ((diff >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); + const uint64_t lo = (diff & 0xffffffffffffffff).convert_to<uint64_t>(); ar.serialize_varint(hi); ar.serialize_varint(lo); if (!ar.stream().good()) diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 007bf265f..553e9951f 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -212,7 +212,7 @@ inline bool do_serialize(Archive &ar, bool &v) * \brief self-explanatory */ #define END_SERIALIZE() \ - return true; \ + return ar.stream().good(); \ } /*! \macro VALUE(f) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 1640fa990..e8b203d9d 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -92,6 +92,8 @@ typedef cryptonote::simple_wallet sw; #define MIN_RING_SIZE 11 // Used to inform user about min ring size -- does not track actual protocol +#define OLD_AGE_WARN_THRESHOLD (30 * 86400 / DIFFICULTY_TARGET_V2) // 30 days + #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ m_auto_refresh_enabled.store(false, std::memory_order_relaxed); \ @@ -4934,7 +4936,7 @@ void simple_wallet::on_new_block(uint64_t height, const cryptonote::block& block m_refresh_progress_reporter.update(height, false); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) +void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) { message_writer(console_color_green, false) << "\r" << tr("Height ") << height << ", " << @@ -4956,6 +4958,8 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, (m_long_payment_id_support ? tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead.") : tr("WARNING: this transaction uses an unencrypted payment ID: these are obsolete. Support will be withdrawn in the future. Use subaddresses instead.")); } } + if (unlock_time) + message_writer() << tr("NOTE: This transaction is locked, see details with: show_transfer ") + epee::string_tools::pod_to_hex(txid); if (m_auto_refresh_refreshing) m_cmd_binder.print_prompt(); else @@ -5606,6 +5610,43 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::prompt_if_old(const std::vector<tools::wallet2::pending_tx> &ptx_vector) +{ + // count the number of old outputs + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (!err.empty()) + return true; + + int max_n_old = 0; + for (const auto &ptx: ptx_vector) + { + int n_old = 0; + for (const auto i: ptx.selected_transfers) + { + const tools::wallet2::transfer_details &td = m_wallet->get_transfer_details(i); + uint64_t age = bc_height - td.m_block_height; + if (age > OLD_AGE_WARN_THRESHOLD) + ++n_old; + } + max_n_old = std::max(max_n_old, n_old); + } + if (max_n_old > 1) + { + std::stringstream prompt; + prompt << tr("Transaction spends more than one very old output. Privacy would be better if they were sent separately."); + prompt << ENDL << tr("Spend them now anyway?"); + std::string accepted = input_line(prompt.str(), true); + if (std::cin.eof()) + return false; + if (!command_line::is_yes(accepted)) + { + return false; + } + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_, bool called_by_mms) { // "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]" @@ -5907,6 +5948,12 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } } + if (!prompt_if_old(ptx_vector)) + { + fail_msg_writer() << tr("transaction cancelled."); + return false; + } + // if more than one tx necessary, prompt user to confirm if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1) { @@ -6090,7 +6137,8 @@ bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, true, args_); + sweep_main(0, true, args_); + return true; } //---------------------------------------------------------------------------------------------------- @@ -6410,6 +6458,12 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st return true; } + if (!prompt_if_old(ptx_vector)) + { + fail_msg_writer() << tr("transaction cancelled."); + return false; + } + // give user total and fee, and prompt to confirm uint64_t total_fee = 0, total_sent = 0; for (size_t n = 0; n < ptx_vector.size(); ++n) @@ -6756,7 +6810,8 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, false, args_); + sweep_main(0, false, args_); + return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_below(const std::vector<std::string> &args_) @@ -6772,7 +6827,8 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - return sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); + sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); + return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector<std::string> &args_) @@ -7683,6 +7739,8 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec local_args.erase(local_args.begin()); } + const uint64_t last_block_height = m_wallet->get_blockchain_current_height(); + if (in || coinbase) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; m_wallet->get_payments(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); @@ -7697,6 +7755,25 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); const std::string type = pd.m_coinbase ? tr("block") : tr("in"); const bool unlocked = m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); + std::string locked_msg = "unlocked"; + if (!unlocked) + { + locked_msg = "locked"; + const uint64_t unlock_time = pd.m_unlock_time; + if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) + { + uint64_t bh = std::max(pd.m_unlock_time, pd.m_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE); + if (bh >= last_block_height) + locked_msg = std::to_string(bh - last_block_height) + " blks"; + } + else + { + uint64_t current_time = static_cast<uint64_t>(time(NULL)); + uint64_t threshold = current_time + (m_wallet->use_fork_rules(2, 0) ? CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V2 : CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V1); + if (threshold < pd.m_unlock_time) + locked_msg = get_human_readable_timespan(std::chrono::seconds(pd.m_unlock_time - threshold)); + } + } transfers.push_back({ type, pd.m_block_height, @@ -7710,7 +7787,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec {{destination, pd.m_amount}}, {pd.m_subaddr_index.minor}, note, - (unlocked) ? "unlocked" : "locked" + locked_msg }); } } diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 76d446ba5..4bf7fa334 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -261,6 +261,7 @@ namespace cryptonote void on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money); std::pair<std::string, std::string> show_outputs_line(const std::vector<uint64_t> &heights, uint64_t blockchain_height, uint64_t highlight_height = std::numeric_limits<uint64_t>::max()) const; bool freeze_thaw(const std::vector<std::string>& args, bool freeze); + bool prompt_if_old(const std::vector<tools::wallet2::pending_tx> &ptx_vector); struct transfer_view { @@ -310,7 +311,7 @@ namespace cryptonote //----------------- i_wallet2_callback --------------------- virtual void on_new_block(uint64_t height, const cryptonote::block& block); - virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); + virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time); virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); diff --git a/src/version.cpp.in b/src/version.cpp.in index 8aaa41b19..28ce38df7 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -1,5 +1,5 @@ #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@" -#define DEF_MONERO_VERSION "0.14.1.0" +#define DEF_MONERO_VERSION "0.14.1.2" #define DEF_MONERO_RELEASE_NAME "Boron Butterfly" #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 032b873d6..1711db482 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -157,7 +157,7 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback } } - virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) + virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) { std::string tx_hash = epee::string_tools::pod_to_hex(txid); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f5b492771..308727657 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -344,7 +344,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl { std::vector<std::vector<uint8_t>> ssl_allowed_fingerprints{ daemon_ssl_allowed_fingerprints.size() }; std::transform(daemon_ssl_allowed_fingerprints.begin(), daemon_ssl_allowed_fingerprints.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); - for (const auto &fpr: daemon_ssl_allowed_fingerprints) + for (const auto &fpr: ssl_allowed_fingerprints) { THROW_WALLET_EXCEPTION_IF(fpr.size() != SSL_FINGERPRINT_SIZE, tools::error::wallet_internal_error, "SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); @@ -399,8 +399,11 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl { const boost::string_ref real_daemon = boost::string_ref{daemon_address}.substr(0, daemon_address.rfind(':')); + /* If SSL or proxy is enabled, then a specific cert, CA or fingerprint must + be specified. This is specific to the wallet. */ const bool verification_required = - ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || use_proxy; + ssl_options.verification != epee::net_utils::ssl_verification_t::none && + (ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || use_proxy); THROW_WALLET_EXCEPTION_IF( verification_required && !ssl_options.has_strong_verification(real_daemon), @@ -1003,7 +1006,7 @@ uint64_t gamma_picker::pick() const uint64_t n_rct = rct_offsets[index] - first_rct; if (n_rct == 0) return std::numeric_limits<uint64_t>::max(); // bad pick - MDEBUG("Picking 1/" << n_rct << " in block " << index); + MTRACE("Picking 1/" << n_rct << " in block " << index); return first_rct + crypto::rand_idx(n_rct); }; @@ -2016,7 +2019,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid); if (0 != m_callback) - m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); + m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, td.m_tx.unlock_time); } total_received_1 += amount; notify = true; @@ -2086,7 +2089,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid); if (0 != m_callback) - m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); + m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, td.m_tx.unlock_time); } total_received_1 += extra_amount; notify = true; @@ -6928,7 +6931,7 @@ uint64_t wallet2::get_base_fee() const else return m_light_wallet_per_kb_fee; } - bool use_dyn_fee = use_fork_rules(HF_VERSION_DYNAMIC_FEE, -720 * 1); + bool use_dyn_fee = use_fork_rules(HF_VERSION_DYNAMIC_FEE, -30 * 1); if (!use_dyn_fee) return FEE_PER_KB; @@ -6959,7 +6962,7 @@ int wallet2::get_fee_algorithm() const return 3; if (use_fork_rules(5, 0)) return 2; - if (use_fork_rules(3, -720 * 14)) + if (use_fork_rules(3, -30 * 14)) return 1; return 0; } @@ -7723,7 +7726,6 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t num_found = 0; // if we have a known ring, use it - bool existing_ring_found = false; if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; @@ -7735,7 +7737,6 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> std::to_string(ring.size()) + ", it cannot be spent now with ring size " + std::to_string(fake_outputs_count + 1) + " as it is smaller: use a higher ring size"); bool own_found = false; - existing_ring_found = true; for (const auto &out: ring) { MINFO("Ring has output " << out); @@ -7761,7 +7762,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } } - if (num_outs <= requested_outputs_count && !existing_ring_found) + if (num_outs <= requested_outputs_count) { for (uint64_t i = 0; i < num_outs; i++) req.outputs.push_back({amount, i}); @@ -7787,6 +7788,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // while we still need more mixins uint64_t num_usable_outs = num_outs; bool allow_blackballed = false; + MDEBUG("Starting gamma picking with " << num_outs << ", num_usable_outs " << num_usable_outs + << ", requested_outputs_count " << requested_outputs_count); while (num_found < requested_outputs_count) { // if we've gone through every possible output, we've gotten all we can @@ -7886,6 +7889,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> picks[type].insert(i); req.outputs.push_back({amount, i}); ++num_found; + MDEBUG("picked " << i << ", " << num_found << " now picked"); } for (const auto &pick: picks) @@ -7978,7 +7982,6 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> outs.back().push_back(std::make_tuple(td.m_global_output_index, boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, mask)); // then pick outs from an existing ring, if any - bool existing_ring_found = false; if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; @@ -11381,12 +11384,13 @@ void wallet2::set_attribute(const std::string &key, const std::string &value) m_attributes[key] = value; } -std::string wallet2::get_attribute(const std::string &key) const +bool wallet2::get_attribute(const std::string &key, std::string &value) const { std::unordered_map<std::string, std::string>::const_iterator i = m_attributes.find(key); if (i == m_attributes.end()) - return std::string(); - return i->second; + return false; + value = i->second; + return true; } void wallet2::set_description(const std::string &description) @@ -11396,7 +11400,10 @@ void wallet2::set_description(const std::string &description) std::string wallet2::get_description() const { - return get_attribute(ATTRIBUTE_DESCRIPTION); + std::string s; + if (get_attribute(ATTRIBUTE_DESCRIPTION, s)) + return s; + return ""; } const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags() @@ -12626,8 +12633,7 @@ std::string wallet2::make_uri(const std::string &address, const std::string &pay if (!payment_id.empty()) { crypto::hash pid32; - crypto::hash8 pid8; - if (!wallet2::parse_long_payment_id(payment_id, pid32) && !wallet2::parse_short_payment_id(payment_id, pid8)) + if (!wallet2::parse_long_payment_id(payment_id, pid32)) { error = "Invalid payment id"; return std::string(); @@ -12721,8 +12727,7 @@ bool wallet2::parse_uri(const std::string &uri, std::string &address, std::strin return false; } crypto::hash hash; - crypto::hash8 hash8; - if (!wallet2::parse_long_payment_id(kv[1], hash) && !wallet2::parse_short_payment_id(kv[1], hash8)) + if (!wallet2::parse_long_payment_id(kv[1], hash)) { error = "Invalid payment id: " + kv[1]; return false; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d101e87f5..30589885e 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -118,7 +118,7 @@ private: public: // Full wallet callbacks virtual void on_new_block(uint64_t height, const cryptonote::block& block) {} - virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} + virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) {} virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) {} @@ -1247,7 +1247,7 @@ private: */ const char* const ATTRIBUTE_DESCRIPTION = "wallet2.description"; void set_attribute(const std::string &key, const std::string &value); - std::string get_attribute(const std::string &key) const; + bool get_attribute(const std::string &key, std::string &value) const; crypto::public_key get_multisig_signer_public_key(const crypto::secret_key &spend_skey) const; crypto::public_key get_multisig_signer_public_key() const; diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 22eaffa6f..9d3605d11 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -66,11 +66,6 @@ namespace const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl = {"rpc-ssl", tools::wallet2::tr("Enable SSL on wallet RPC connections: enabled|disabled|autodetect"), "autodetect"}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl_private_key = {"rpc-ssl-private-key", tools::wallet2::tr("Path to a PEM format private key"), ""}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl_certificate = {"rpc-ssl-certificate", tools::wallet2::tr("Path to a PEM format certificate"), ""}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl_ca_certificates = {"rpc-ssl-ca-certificates", tools::wallet2::tr("Path to file containing concatenated PEM format certificate(s) to replace system CA(s).")}; - const command_line::arg_descriptor<std::vector<std::string>> arg_rpc_ssl_allowed_fingerprints = {"rpc-ssl-allowed-fingerprints", tools::wallet2::tr("List of certificate fingerprints to allow")}; constexpr const char default_rpc_username[] = "monero"; @@ -244,45 +239,6 @@ namespace tools assert(bool(http_login)); } // end auth enabled - auto rpc_ssl_private_key = command_line::get_arg(vm, arg_rpc_ssl_private_key); - auto rpc_ssl_certificate = command_line::get_arg(vm, arg_rpc_ssl_certificate); - auto rpc_ssl_ca_file = command_line::get_arg(vm, arg_rpc_ssl_ca_certificates); - auto rpc_ssl_allowed_fingerprints = command_line::get_arg(vm, arg_rpc_ssl_allowed_fingerprints); - auto rpc_ssl = command_line::get_arg(vm, arg_rpc_ssl); - epee::net_utils::ssl_options_t rpc_ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; - - if (!rpc_ssl_ca_file.empty() || !rpc_ssl_allowed_fingerprints.empty()) - { - std::vector<std::vector<uint8_t>> allowed_fingerprints{ rpc_ssl_allowed_fingerprints.size() }; - std::transform(rpc_ssl_allowed_fingerprints.begin(), rpc_ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); - for (const auto &fpr: rpc_ssl_allowed_fingerprints) - { - if (fpr.size() != SSL_FINGERPRINT_SIZE) - { - MERROR("SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); - return false; - } - } - - rpc_ssl_options = epee::net_utils::ssl_options_t{ - std::move(allowed_fingerprints), std::move(rpc_ssl_ca_file) - }; - } - - // user specified CA file or fingeprints implies enabled SSL by default - if (rpc_ssl_options.verification != epee::net_utils::ssl_verification_t::user_certificates || !command_line::is_arg_defaulted(vm, arg_rpc_ssl)) - { - if (!epee::net_utils::ssl_support_from_string(rpc_ssl_options.support, rpc_ssl)) - { - MERROR("Invalid argument for " << std::string(arg_rpc_ssl.name)); - return false; - } - } - - rpc_ssl_options.auth = epee::net_utils::ssl_authentication_t{ - std::move(rpc_ssl_private_key), std::move(rpc_ssl_certificate) - }; - m_auto_refresh_period = DEFAULT_AUTO_REFRESH_PERIOD; m_last_auto_refresh_time = boost::posix_time::min_date_time; @@ -292,7 +248,7 @@ namespace tools auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( rng, std::move(bind_port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), - std::move(rpc_ssl_options) + std::move(rpc_config->ssl_options) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -394,7 +350,7 @@ namespace tools entry.destinations.push_back(wallet_rpc::transfer_destination()); wallet_rpc::transfer_destination &td = entry.destinations.back(); td.amount = d.amount; - td.address = get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr); + td.address = d.original.empty() ? get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) : d.original; } entry.type = "out"; @@ -419,6 +375,14 @@ namespace tools entry.amount = pd.m_amount_in - pd.m_change - entry.fee; entry.unlock_time = pd.m_tx.unlock_time; entry.note = m_wallet->get_tx_note(txid); + + for (const auto &d: pd.m_dests) { + entry.destinations.push_back(wallet_rpc::transfer_destination()); + wallet_rpc::transfer_destination &td = entry.destinations.back(); + td.amount = d.amount; + td.address = d.original.empty() ? get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) : d.original; + } + entry.type = is_failed ? "failed" : "pending"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; for (uint32_t i: pd.m_subaddr_indices) @@ -1781,6 +1745,11 @@ namespace tools else if (payment_id_str.size() == 2 * sizeof(payment_id8)) { r = epee::string_tools::hex_to_pod(payment_id_str, payment_id8); + if (r) + { + memcpy(payment_id.data, payment_id8.data, 8); + memset(payment_id.data + 8, 0, 24); + } } else { @@ -1842,14 +1811,12 @@ namespace tools wallet2::transfer_container transfers; m_wallet->get_transfers(transfers); - bool transfers_found = false; for (const auto& td : transfers) { if (!filter || available != td.m_spent) { if (req.account_index != td.m_subaddr_index.major || (!req.subaddr_indices.empty() && req.subaddr_indices.count(td.m_subaddr_index.minor) == 0)) continue; - transfers_found = true; wallet_rpc::transfer_details rpc_transfers; rpc_transfers.amount = td.amount(); rpc_transfers.spent = td.m_spent; @@ -2135,7 +2102,12 @@ namespace tools return false; } - res.value = m_wallet->get_attribute(req.key); + if (!m_wallet->get_attribute(req.key, res.value)) + { + er.code = WALLET_RPC_ERROR_CODE_ATTRIBUTE_NOT_FOUND; + er.message = "Attribute not found."; + return false; + } return true; } bool wallet_rpc_server::on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx) @@ -2814,20 +2786,20 @@ namespace tools } crypto::hash long_payment_id; - crypto::hash8 short_payment_id; if (!wallet2::parse_long_payment_id(req.payment_id, payment_id)) { if (!wallet2::parse_short_payment_id(req.payment_id, info.payment_id)) { er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 16 or 64 character string"; + er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 64 character string"; return false; } else { - memcpy(payment_id.data, info.payment_id.data, 8); - memset(payment_id.data + 8, 0, 24); + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment id has invalid format: standalone short payment IDs are forbidden, they must be part of an integrated address"; + return false; } } } @@ -4159,7 +4131,11 @@ namespace tools std::move(req.ssl_private_key_path), std::move(req.ssl_certificate_path) }; - if (ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled && !ssl_options.has_strong_verification(boost::string_ref{})) + const bool verification_required = + ssl_options.verification != epee::net_utils::ssl_verification_t::none && + ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled; + + if (verification_required && !ssl_options.has_strong_verification(boost::string_ref{})) { er.code = WALLET_RPC_ERROR_CODE_NO_DAEMON_CONNECTION; er.message = "SSL is enabled but no user certificate or fingerprints were provided"; @@ -4404,11 +4380,6 @@ int main(int argc, char** argv) { command_line::add_arg(desc_params, arg_from_json); command_line::add_arg(desc_params, arg_wallet_dir); command_line::add_arg(desc_params, arg_prompt_for_password); - command_line::add_arg(desc_params, arg_rpc_ssl); - command_line::add_arg(desc_params, arg_rpc_ssl_private_key); - command_line::add_arg(desc_params, arg_rpc_ssl_certificate); - command_line::add_arg(desc_params, arg_rpc_ssl_ca_certificates); - command_line::add_arg(desc_params, arg_rpc_ssl_allowed_fingerprints); daemonizer::init_options(hidden_options, desc_params); desc_params.add(hidden_options); diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index 011d146d4..9434fbc3e 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -75,3 +75,4 @@ #define WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED -42 #define WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC -43 #define WALLET_RPC_ERROR_CODE_INVALID_LOG_LEVEL -44 +#define WALLET_RPC_ERROR_CODE_ATTRIBUTE_NOT_FOUND -45 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bbb0bc051..afc69ee88 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -66,6 +66,7 @@ else () # Emulate the FindGTest module's variable. set(GTEST_LIBRARIES gtest gtest_main) + set(GTEST_BOTH_LIBRARIES gtest gtest_main) include_directories(SYSTEM "${CMAKE_CURRENT_SOURCE_DIR}/gtest/include") endif (GTest_FOUND) diff --git a/tests/README.md b/tests/README.md index 053dd2244..f6dd25ead 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,7 +2,7 @@ To run all tests, run: -``` +```bash cd /path/to/monero make [-jn] debug-test # where n is number of compiler processes ``` @@ -17,7 +17,7 @@ Tests are located in `tests/core_tests/`, and follow a straightforward naming co To run only Monero's core tests (after building): -``` +```bash cd build/debug/tests/core_tests ctest ``` @@ -36,7 +36,7 @@ Tests correspond to components under `src/crypto/`. A quick comparison reveals t To run only Monero's crypto tests (after building): -``` +```bash cd build/debug/tests/crypto ctest ``` @@ -53,13 +53,13 @@ To run the same tests on a release build, replace `debug` with `release`. Functional tests are located under the `tests/functional` directory. First, run a regtest daemon in the offline mode and with a fixed difficulty: -``` +```bash monerod --regtest --offline --fixed-difficulty 1 ``` Alternatively, you can run multiple daemons and let them connect with each other by using `--add-exclusive-node`. In this case, make sure that the same fixed difficulty is given to all the daemons. Next, restore a mainnet wallet with the following seed and restore height 0 (the file path doesn't matter): -``` +```bash velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted ``` @@ -77,7 +77,7 @@ Hash tests exist under `tests/hash`, and include a set of target hashes in text To run only Monero's hash tests (after building): -``` +```bash cd build/debug/tests/hash ctest ``` @@ -98,7 +98,7 @@ Performance tests are located in `tests/performance_tests`, and test features fo To run only Monero's performance tests (after building): -``` +```bash cd build/debug/tests/performance_tests ./performance_tests ``` @@ -115,7 +115,7 @@ Unit tests are defined under the `tests/unit_tests` directory. Independent compo To run only Monero's unit tests (after building): -``` +```bash cd build/debug/tests/unit_tests ctest ``` diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 614585349..8dde1f475 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -546,6 +546,7 @@ void block_tracker::global_indices(const cryptonote::transaction *tx, std::vecto void block_tracker::get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs){ auto & vct = m_outs[amount]; const size_t n_outs = vct.size(); + CHECK_AND_ASSERT_THROW_MES(n_outs > 0, "n_outs is 0"); std::set<size_t> used; std::vector<size_t> choices; diff --git a/tests/core_tests/wallet_tools.cpp b/tests/core_tests/wallet_tools.cpp index d9cee34c1..21a9455c0 100644 --- a/tests/core_tests/wallet_tools.cpp +++ b/tests/core_tests/wallet_tools.cpp @@ -163,6 +163,7 @@ bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptono void wallet_tools::gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt) { + CHECK_AND_ASSERT_THROW_MES(mixin != 0, "mixin is zero"); src.amount = td.amount(); src.rct = td.is_rct(); diff --git a/tests/difficulty/CMakeLists.txt b/tests/difficulty/CMakeLists.txt index fb0dd6b9e..c4f5a5dc0 100644 --- a/tests/difficulty/CMakeLists.txt +++ b/tests/difficulty/CMakeLists.txt @@ -36,7 +36,7 @@ add_executable(difficulty-tests ${difficulty_headers}) target_link_libraries(difficulty-tests PRIVATE - cryptonote_core + cryptonote_basic ${EXTRA_LIBRARIES}) set_property(TARGET difficulty-tests PROPERTY diff --git a/tests/difficulty/difficulty.cpp b/tests/difficulty/difficulty.cpp index 11ce0bd73..18f1d0030 100644 --- a/tests/difficulty/difficulty.cpp +++ b/tests/difficulty/difficulty.cpp @@ -124,7 +124,7 @@ int main(int argc, char *argv[]) { cryptonote::difficulty_type wide_res = cryptonote::next_difficulty( std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end), std::vector<cryptonote::difficulty_type>(wide_cumulative_difficulties.begin() + begin, wide_cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET); - if (wide_res.convert_to<uint64_t>() != res) { + if ((wide_res & 0xffffffffffffffff).convert_to<uint64_t>() != res) { cerr << "Wrong wide difficulty for block " << n << endl << "Expected: " << res << endl << "Found: " << wide_res << endl; diff --git a/tests/functional_tests/bans.py b/tests/functional_tests/bans.py index bb3051a6a..e859e58c9 100755 --- a/tests/functional_tests/bans.py +++ b/tests/functional_tests/bans.py @@ -28,6 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function import time """Test peer baning RPC calls @@ -42,7 +43,7 @@ from framework.daemon import Daemon class BanTest(): def run_test(self): - print 'Testing bans' + print('Testing bans') daemon = Daemon() res = daemon.get_bans() diff --git a/tests/functional_tests/blockchain.py b/tests/functional_tests/blockchain.py index 6376b86d0..2bd7672c0 100755 --- a/tests/functional_tests/blockchain.py +++ b/tests/functional_tests/blockchain.py @@ -28,6 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function import time """Test daemon blockchain RPC calls @@ -50,7 +51,7 @@ class BlockchainTest(): self._test_alt_chains() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() @@ -58,7 +59,7 @@ class BlockchainTest(): def _test_generateblocks(self, blocks): assert blocks >= 2 - print "Test generating", blocks, 'blocks' + print("Test generating", blocks, 'blocks') daemon = Daemon() @@ -73,7 +74,7 @@ class BlockchainTest(): # we should not see a block at height ok = False - try: daemon.getblock(height) + try: daemon.getblock(height = height) except: ok = True assert ok @@ -91,7 +92,7 @@ class BlockchainTest(): # get the blocks, check they have the right height res_getblock = [] for n in range(blocks): - res_getblock.append(daemon.getblock(height + n)) + res_getblock.append(daemon.getblock(height = height + n)) block_header = res_getblock[n].block_header assert abs(block_header.timestamp - time.time()) < 10 # within 10 seconds assert block_header.height == height + n @@ -110,7 +111,7 @@ class BlockchainTest(): # we should not see a block after that ok = False - try: daemon.getblock(height + blocks) + try: daemon.getblock(height = height + blocks) except: ok = True assert ok @@ -156,7 +157,7 @@ class BlockchainTest(): # we should not see the popped block anymore ok = False - try: daemon.getblock(height + blocks - 1) + try: daemon.getblock(height = height + blocks - 1) except: ok = True assert ok @@ -182,14 +183,14 @@ class BlockchainTest(): for idx in tx.output_indices: assert idx == running_output_index running_output_index += 1 - res_out = daemon.get_outs([{'amount': 0, 'index': i} for i in tx.output_indices], get_txid = True) + res_out = daemon.get_outs([{'amount': 0, 'index': idx} for idx in tx.output_indices], get_txid = True) assert len(res_out.outs) == len(tx.output_indices) for out in res_out.outs: assert len(out.key) == 64 assert len(out.mask) == 64 assert not out.unlocked - assert out.height == i + 1 - assert out.txid == txids[i + 1] + assert out.height == i + assert out.txid == txids[i] for i in range(height + nblocks - 1): res_sum = daemon.get_coinbase_tx_sum(i, 1) @@ -255,7 +256,7 @@ class BlockchainTest(): alt_blocks[i] = txid nonce += 1 - print 'mining 3 on 1' + print('mining 3 on 1') # three more on [1] chain1 = [] res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 3, prev_block = alt_blocks[1], starting_nonce = nonce) @@ -275,7 +276,7 @@ class BlockchainTest(): for txid in alt_blocks: assert txid in res.blks_hashes or txid == alt_blocks[1] - print 'mining 4 on 3' + print('mining 4 on 3') # 4 more on [3], the chain will reorg when we mine the 4th top_block_hash = blk_hash prev_block = alt_blocks[3] diff --git a/tests/functional_tests/cold_signing.py b/tests/functional_tests/cold_signing.py index 59a879e0a..a722d8927 100755 --- a/tests/functional_tests/cold_signing.py +++ b/tests/functional_tests/cold_signing.py @@ -28,11 +28,10 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time - """Test cold tx signing """ +from __future__ import print_function from framework.daemon import Daemon from framework.wallet import Wallet @@ -44,13 +43,13 @@ class ColdSigningTest(): self.transfer() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() def create(self, idx): - print 'Creating hot and cold wallet' + print('Creating hot and cold wallet') self.hot_wallet = Wallet(idx = 0) # close the wallet if any, will throw if none is loaded @@ -116,7 +115,7 @@ class ColdSigningTest(): assert len(res.unsigned_txset) > 0 unsigned_txset = res.unsigned_txset - print 'Signing transaction with cold wallet' + print('Signing transaction with cold wallet') res = self.cold_wallet.describe_transfer(unsigned_txset = unsigned_txset) assert len(res.desc) == 1 desc = res.desc[0] @@ -140,7 +139,7 @@ class ColdSigningTest(): txid = res.tx_hash_list[0] assert len(txid) == 64 - print 'Submitting transaction with hot wallet' + print('Submitting transaction with hot wallet') res = self.hot_wallet.submit_transfer(signed_txset) assert len(res.tx_hash_list) > 0 assert res.tx_hash_list[0] == txid diff --git a/tests/functional_tests/daemon_info.py b/tests/functional_tests/daemon_info.py index bd3528c3f..4fa768b03 100755 --- a/tests/functional_tests/daemon_info.py +++ b/tests/functional_tests/daemon_info.py @@ -36,6 +36,7 @@ Test the following RPCs: """ +from __future__ import print_function from framework.daemon import Daemon class DaemonGetInfoTest(): diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index e754b4e33..25ab641ab 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -10,7 +10,7 @@ import string import os USAGE = 'usage: functional_tests_rpc.py <python> <srcdir> <builddir> [<tests-to-run> | all]' -DEFAULT_TESTS = ['daemon_info', 'blockchain', 'wallet_address', 'integrated_address', 'mining', 'transfer', 'txpool', 'multisig', 'cold_signing', 'sign_message', 'proofs'] +DEFAULT_TESTS = ['bans', 'daemon_info', 'blockchain', 'wallet_address', 'integrated_address', 'mining', 'transfer', 'txpool', 'multisig', 'cold_signing', 'sign_message', 'proofs', 'get_output_distribution'] try: python = sys.argv[1] srcdir = sys.argv[2] diff --git a/tests/functional_tests/get_output_distribution.py b/tests/functional_tests/get_output_distribution.py new file mode 100755 index 000000000..93822e90a --- /dev/null +++ b/tests/functional_tests/get_output_distribution.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test get_output_distribution RPC +""" + +from __future__ import print_function +from framework.daemon import Daemon +from framework.wallet import Wallet + +class GetOutputDistributionTest(): + def run_test(self): + self.reset() + self.create() + self.test_get_output_distribution() + + def reset(self): + print('Resetting blockchain') + daemon = Daemon() + daemon.pop_blocks(1000) + daemon.flush_txpool() + + def create(self): + self.wallet = Wallet() + # close the wallet if any, will throw if none is loaded + try: self.wallet.close_wallet() + except: pass + res = self.wallet.restore_deterministic_wallet(seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted') + + def test_get_output_distribution(self): + print("Test get_output_distribution") + + daemon = Daemon() + + res = daemon.get_output_distribution([0], 0, 0) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 1 + assert d.distribution[0] == 0 + + res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1) + + res = daemon.get_output_distribution([0], 0, 0) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 2 + assert d.distribution[0] == 0 + assert d.distribution[1] == 1 + + res = daemon.pop_blocks(1) + + res = daemon.get_output_distribution([0], 0, 0) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 1 + assert d.distribution[0] == 0 + + res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 3) + + res = daemon.get_output_distribution([0], 0, 0, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 4 + assert d.distribution[0] == 0 + assert d.distribution[1] == 1 + assert d.distribution[2] == 2 + assert d.distribution[3] == 3 + + # extend + res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80) + + res = daemon.get_output_distribution([0], 0, 0, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 84 + for h in range(len(d.distribution)): + assert d.distribution[h] == h + + # pop and replace, this will do through the "trim and extend" path + res = daemon.pop_blocks(2) + self.wallet.refresh() + dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000} + self.wallet.transfer([dst]) + res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1) + for step in range(3): # the second will be cached, the third will also be cached, but we get it in non-cumulative mode + res = daemon.get_output_distribution([0], 0, 0, cumulative = step < 3) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 83 + for h in range(len(d.distribution)): + assert d.distribution[h] == (h if step < 3 else 1) + (2 if h == len(d.distribution) - 1 else 0) + + # start at 0, end earlier + res = daemon.get_output_distribution([0], 0, 40, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 41 + for h in range(len(d.distribution)): + assert d.distribution[h] == h + + # start after 0, end earlier + res = daemon.get_output_distribution([0], 10, 20, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 9 + assert d.binary == False + assert len(d.distribution) == 11 + for h in range(len(d.distribution)): + assert d.distribution[h] == 10 + h + + # straddling up + res = daemon.get_output_distribution([0], 15, 25, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 14 + assert d.binary == False + assert len(d.distribution) == 11 + for h in range(len(d.distribution)): + assert d.distribution[h] == 15 + h + + # straddling down + res = daemon.get_output_distribution([0], 8, 18, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 7 + assert d.binary == False + assert len(d.distribution) == 11 + for h in range(len(d.distribution)): + assert d.distribution[h] == 8 + h + + # encompassing + res = daemon.get_output_distribution([0], 5, 20, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 4 + assert d.binary == False + assert len(d.distribution) == 16 + for h in range(len(d.distribution)): + assert d.distribution[h] == 5 + h + + # single + res = daemon.get_output_distribution([0], 2, 2, cumulative = True) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 0 + assert d.base == 1 + assert d.binary == False + assert len(d.distribution) == 1 + assert d.distribution[0] == 2 + + # a non existent amount + res = daemon.get_output_distribution([1], 0, 0) + assert len(res.distributions) == 1 + d = res.distributions[0] + assert d.amount == 1 + assert d.base == 0 + assert d.binary == False + assert len(d.distribution) == 83 + for h in range(len(d.distribution)): + assert d.distribution[h] == 0 + + +class Guard: + def __enter__(self): + for i in range(4): + Wallet(idx = i).auto_refresh(False) + def __exit__(self, exc_type, exc_value, traceback): + for i in range(4): + Wallet(idx = i).auto_refresh(True) + +if __name__ == '__main__': + with Guard() as guard: + GetOutputDistributionTest().run_test() diff --git a/tests/functional_tests/integrated_address.py b/tests/functional_tests/integrated_address.py index 338dd14ae..4e42261a6 100755 --- a/tests/functional_tests/integrated_address.py +++ b/tests/functional_tests/integrated_address.py @@ -28,8 +28,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time - """Test integrated address RPC calls Test the following RPCs: @@ -38,6 +36,7 @@ Test the following RPCs: """ +from __future__ import print_function from framework.wallet import Wallet class IntegratedAddressTest(): @@ -46,7 +45,7 @@ class IntegratedAddressTest(): self.check() def create(self): - print 'Creating wallet' + print('Creating wallet') wallet = Wallet() # close the wallet if any, will throw if none is loaded try: wallet.close_wallet() @@ -59,7 +58,7 @@ class IntegratedAddressTest(): def check(self): wallet = Wallet() - print 'Checking local address' + print('Checking local address') res = wallet.make_integrated_address(payment_id = '0123456789abcdef') assert res.integrated_address == '4CMe2PUhs4J4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfSbLRB61BQVATzerHGj' assert res.payment_id == '0123456789abcdef' @@ -67,7 +66,7 @@ class IntegratedAddressTest(): assert res.standard_address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' assert res.payment_id == '0123456789abcdef' - print 'Checking different address' + print('Checking different address') res = wallet.make_integrated_address(standard_address = '46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', payment_id = '1122334455667788') assert res.integrated_address == '4GYjoMG9Y2BBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCVSs1ZojwrDCGS5rUuo' assert res.payment_id == '1122334455667788' @@ -75,7 +74,7 @@ class IntegratedAddressTest(): assert res.standard_address == '46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK' assert res.payment_id == '1122334455667788' - print 'Checking bad payment id' + print('Checking bad payment id') fails = 0 try: wallet.make_integrated_address(standard_address = '46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', payment_id = '11223344556677880') except: fails += 1 @@ -89,7 +88,7 @@ class IntegratedAddressTest(): except: fails += 1 assert fails == 5 - print 'Checking bad standard address' + print('Checking bad standard address') fails = 0 try: wallet.make_integrated_address(standard_address = '46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerr', payment_id = '1122334455667788') except: fails += 1 diff --git a/tests/functional_tests/mining.py b/tests/functional_tests/mining.py index 78dc68640..5c14d34fd 100755 --- a/tests/functional_tests/mining.py +++ b/tests/functional_tests/mining.py @@ -28,6 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function import time """Test daemon mining RPC calls @@ -48,13 +49,13 @@ class MiningTest(): self.mine() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() def create(self): - print 'Creating wallet' + print('Creating wallet') wallet = Wallet() # close the wallet if any, will throw if none is loaded try: wallet.close_wallet() @@ -62,7 +63,7 @@ class MiningTest(): res = wallet.restore_deterministic_wallet(seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted') def mine(self): - print "Test mining" + print("Test mining") daemon = Daemon() wallet = Wallet() diff --git a/tests/functional_tests/multisig.py b/tests/functional_tests/multisig.py index 476e3a02d..3c8cd9c1d 100755 --- a/tests/functional_tests/multisig.py +++ b/tests/functional_tests/multisig.py @@ -28,7 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time +from __future__ import print_function """Test multisig transfers """ @@ -70,7 +70,7 @@ class MultisigTest(): self.check_transaction(txid) def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() diff --git a/tests/functional_tests/proofs.py b/tests/functional_tests/proofs.py index 844131095..243929dc3 100755 --- a/tests/functional_tests/proofs.py +++ b/tests/functional_tests/proofs.py @@ -28,7 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time +from __future__ import print_function """Test misc proofs (tx key, send, receive, reserve) """ @@ -47,7 +47,7 @@ class ProofsTest(): self.check_reserve_proof() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() diff --git a/tests/functional_tests/sign_message.py b/tests/functional_tests/sign_message.py index 4c3ec3588..de8f0cee2 100755 --- a/tests/functional_tests/sign_message.py +++ b/tests/functional_tests/sign_message.py @@ -28,7 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time +from __future__ import print_function """Test message signing/verification RPC calls @@ -46,7 +46,7 @@ class MessageSigningTest(): self.check_signing() def create(self): - print 'Creating wallets' + print('Creating wallets') seeds = [ 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted', 'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout', @@ -66,7 +66,7 @@ class MessageSigningTest(): assert res.seed == seeds[i] def check_signing(self): - print 'Signing/verifing messages' + print('Signing/verifing messages') messages = ['foo', ''] for message in messages: res = self.wallet[0].sign(message) diff --git a/tests/functional_tests/speed.py b/tests/functional_tests/speed.py index bd8892df8..ed1e332e9 100755 --- a/tests/functional_tests/speed.py +++ b/tests/functional_tests/speed.py @@ -40,7 +40,7 @@ Test the following RPCs: import time from time import sleep -from decimal import Decimal +from __future__ import print_function from framework.daemon import Daemon from framework.wallet import Wallet diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py index 1ff641d1f..d9a6e592e 100755 --- a/tests/functional_tests/transfer.py +++ b/tests/functional_tests/transfer.py @@ -28,7 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time +from __future__ import print_function import json """Test simple transfers @@ -48,13 +48,13 @@ class TransferTest(): self.sweep_single() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() def create(self): - print 'Creating wallets' + print('Creating wallets') seeds = [ 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted', 'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout', @@ -297,7 +297,7 @@ class TransferTest(): assert res.unlocked_balance <= res.balance assert res.blocks_to_unlock == 9 - print 'Creating multi out transfer' + print('Creating multi out transfer') self.wallet[0].refresh() @@ -519,6 +519,9 @@ class TransferTest(): res = self.wallet[2].get_bulk_payments(payment_ids = ['1'*64, '1234500000012345abcde00000abcdeff1234500000012345abcde00000abcde', '2'*64]) assert len(res.payments) >= 1 # one tx was sent + res = self.wallet[1].get_bulk_payments(["1111111122222222"]) + assert len(res.payments) >= 1 # we have one of these + def check_double_spend_detection(self): print('Checking double spend detection') txes = [[None, None], [None, None]] diff --git a/tests/functional_tests/txpool.py b/tests/functional_tests/txpool.py index d74395f10..b6af4c84f 100755 --- a/tests/functional_tests/txpool.py +++ b/tests/functional_tests/txpool.py @@ -28,7 +28,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time +from __future__ import print_function """Test txpool """ @@ -44,13 +44,13 @@ class TransferTest(): self.check_txpool() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() def create(self): - print 'Creating wallet' + print('Creating wallet') wallet = Wallet() # close the wallet if any, will throw if none is loaded try: wallet.close_wallet() @@ -114,15 +114,16 @@ class TransferTest(): assert sorted(res.tx_hashes) == sorted(txes.keys()) print('Flushing 2 transactions') - daemon.flush_txpool([txes.keys()[1], txes.keys()[3]]) + txes_keys = list(txes.keys()) + daemon.flush_txpool([txes_keys[1], txes_keys[3]]) res = daemon.get_transaction_pool() assert len(res.transactions) == txpool_size - 2 - assert len([x for x in res.transactions if x.id_hash == txes.keys()[1]]) == 0 - assert len([x for x in res.transactions if x.id_hash == txes.keys()[3]]) == 0 + assert len([x for x in res.transactions if x.id_hash == txes_keys[1]]) == 0 + assert len([x for x in res.transactions if x.id_hash == txes_keys[3]]) == 0 - new_keys = txes.keys() - new_keys.remove(txes.keys()[1]) - new_keys.remove(txes.keys()[3]) + new_keys = list(txes.keys()) + new_keys.remove(txes_keys[1]) + new_keys.remove(txes_keys[3]) res = daemon.get_transaction_pool_hashes() assert sorted(res.tx_hashes) == sorted(new_keys) diff --git a/tests/functional_tests/wallet_address.py b/tests/functional_tests/wallet_address.py index 8a55521c6..4ff059a6f 100755 --- a/tests/functional_tests/wallet_address.py +++ b/tests/functional_tests/wallet_address.py @@ -29,8 +29,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time - """Test transaction creation RPC calls Test the following RPCs: @@ -38,6 +36,7 @@ Test the following RPCs: """ +from __future__ import print_function from framework.wallet import Wallet from framework.daemon import Daemon @@ -52,13 +51,13 @@ class WalletAddressTest(): self.languages() def reset(self): - print 'Resetting blockchain' + print('Resetting blockchain') daemon = Daemon() daemon.pop_blocks(1000) daemon.flush_txpool() def create(self): - print 'Creating wallet' + print('Creating wallet') wallet = Wallet() # close the wallet if any, will throw if none is loaded try: wallet.close_wallet() @@ -69,7 +68,7 @@ class WalletAddressTest(): assert res.seed == seed def check_main_address(self): - print 'Getting address' + print('Getting address') wallet = Wallet() res = wallet.get_address() assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', res @@ -79,7 +78,7 @@ class WalletAddressTest(): assert res.addresses[0].used == False def check_keys(self): - print 'Checking keys' + print('Checking keys') wallet = Wallet() res = wallet.query_key('view_key') assert res.key == '49774391fa5e8d249fc2c5b45dadef13534bf2483dede880dac88f061e809100' @@ -89,7 +88,7 @@ class WalletAddressTest(): assert res.key == 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' def create_subaddresses(self): - print 'Creating subaddresses' + print('Creating subaddresses') wallet = Wallet() res = wallet.create_account("idx1") assert res.account_index == 1, res @@ -160,7 +159,7 @@ class WalletAddressTest(): assert res.index == {'major': 1, 'minor': 0} def open_close(self): - print 'Testing open/close' + print('Testing open/close') wallet = Wallet() res = wallet.get_address() @@ -200,7 +199,7 @@ class WalletAddressTest(): except: pass languages = res.languages for language in languages: - print 'Creating ' + str(language) + ' wallet' + print('Creating ' + str(language) + ' wallet') wallet.create_wallet(filename = '', language = language) res = wallet.query_key('mnemonic') wallet.close_wallet() diff --git a/tests/gtest/README.md b/tests/gtest/README.md index e0ea1b0f3..43a16bde0 100644 --- a/tests/gtest/README.md +++ b/tests/gtest/README.md @@ -14,15 +14,19 @@ Suppose you put Google Test in directory `${GTEST_DIR}`. To build it, create a library build target (or a project as called by Visual Studio and Xcode) to compile - ${GTEST_DIR}/src/gtest-all.cc +```bash +${GTEST_DIR}/src/gtest-all.cc +``` with `${GTEST_DIR}/include` in the system header search path and `${GTEST_DIR}` in the normal header search path. Assuming a Linux-like system and gcc, something like the following will do: - g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \ - -pthread -c ${GTEST_DIR}/src/gtest-all.cc - ar -rv libgtest.a gtest-all.o +```bash +g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \ + -pthread -c ${GTEST_DIR}/src/gtest-all.cc +ar -rv libgtest.a gtest-all.o +``` (We need `-pthread` as Google Test uses threads.) @@ -30,8 +34,10 @@ Next, you should compile your test source file with `${GTEST_DIR}/include` in the system header search path, and link it with gtest and any other necessary libraries: - g++ -isystem ${GTEST_DIR}/include -pthread path/to/your_test.cc libgtest.a \ - -o your_test +```bash +g++ -isystem ${GTEST_DIR}/include -pthread path/to/your_test.cc libgtest.a \ + -o your_test +``` As an example, the make/ directory contains a Makefile that you can use to build Google Test on systems where GNU make is available @@ -43,9 +49,11 @@ script. If the default settings are correct for your environment, the following commands should succeed: - cd ${GTEST_DIR}/make - make - ./sample1_unittest +```bash +cd ${GTEST_DIR}/make +make +./sample1_unittest +``` If you see errors, try to tweak the contents of `make/Makefile` to make them go away. There are instructions in `make/Makefile` on how to do @@ -62,14 +70,18 @@ CMake works by generating native makefiles or build projects that can be used in the compiler environment of your choice. The typical workflow starts with: - mkdir mybuild # Create a directory to hold the build output. - cd mybuild - cmake ${GTEST_DIR} # Generate native build scripts. +```bash +mkdir mybuild # Create a directory to hold the build output. +cd mybuild +cmake ${GTEST_DIR} # Generate native build scripts. +``` If you want to build Google Test's samples, you should replace the last command with - cmake -Dgtest_build_samples=ON ${GTEST_DIR} +```bash +cmake -Dgtest_build_samples=ON ${GTEST_DIR} +``` If you are on a \*nix system, you should now see a Makefile in the current directory. Just type 'make' to build gtest. @@ -108,7 +120,9 @@ end up in your selected build directory (selected in the Xcode "Preferences..." -> "Building" pane and defaults to xcode/build). Alternatively, at the command line, enter: - xcodebuild +```bash +xcodebuild +``` This will build the "Release" configuration of gtest.framework in your default build location. See the "xcodebuild" man page for more @@ -152,18 +166,24 @@ tell Google Test to use the same TR1 tuple library the rest of your project uses, or the two tuple implementations will clash. To do that, add - -DGTEST_USE_OWN_TR1_TUPLE=0 +```bash +-DGTEST_USE_OWN_TR1_TUPLE=0 +``` to the compiler flags while compiling Google Test and your tests. If you want to force Google Test to use its own tuple library, just add - -DGTEST_USE_OWN_TR1_TUPLE=1 +```bash +-DGTEST_USE_OWN_TR1_TUPLE=1 +``` to the compiler flags instead. If you don't want Google Test to use tuple at all, add - -DGTEST_HAS_TR1_TUPLE=0 +```bash +-DGTEST_HAS_TR1_TUPLE=0 +``` and all features using tuple will be disabled. @@ -177,11 +197,15 @@ macro to see whether this is the case (yes if the macro is `#defined` to If Google Test doesn't correctly detect whether pthread is available in your environment, you can force it with - -DGTEST_HAS_PTHREAD=1 +```bash +-DGTEST_HAS_PTHREAD=1 +``` or - -DGTEST_HAS_PTHREAD=0 +```bash +-DGTEST_HAS_PTHREAD=0 +``` When Google Test uses pthread, you may need to add flags to your compiler and/or linker to select the pthread library, or you'll get @@ -198,7 +222,9 @@ as a shared library (known as a DLL on Windows) if you prefer. To compile *gtest* as a shared library, add - -DGTEST_CREATE_SHARED_LIBRARY=1 +```bash +-DGTEST_CREATE_SHARED_LIBRARY=1 +``` to the compiler flags. You'll also need to tell the linker to produce a shared library instead - consult your linker's manual for how to do @@ -206,7 +232,9 @@ it. To compile your *tests* that use the gtest shared library, add - -DGTEST_LINKED_AS_SHARED_LIBRARY=1 +```bash +-DGTEST_LINKED_AS_SHARED_LIBRARY=1 +``` to the compiler flags. @@ -229,18 +257,24 @@ conflict. Specifically, if both Google Test and some other code define macro FOO, you can add - -DGTEST_DONT_DEFINE_FOO=1 +```bash +-DGTEST_DONT_DEFINE_FOO=1 +``` to the compiler flags to tell Google Test to change the macro's name from `FOO` to `GTEST_FOO`. Currently `FOO` can be `FAIL`, `SUCCEED`, or `TEST`. For example, with `-DGTEST_DONT_DEFINE_TEST=1`, you'll need to write - GTEST_TEST(SomeTest, DoesThis) { ... } +```c++ +GTEST_TEST(SomeTest, DoesThis) { ... } +``` instead of - TEST(SomeTest, DoesThis) { ... } +```c++ +TEST(SomeTest, DoesThis) { ... } +``` in order to define a test. @@ -254,9 +288,11 @@ To make sure your changes work as intended and don't break existing functionality, you'll want to compile and run Google Test's own tests. For that you can use CMake: - mkdir mybuild - cd mybuild - cmake -Dgtest_build_tests=ON ${GTEST_DIR} +```bash +mkdir mybuild +cd mybuild +cmake -Dgtest_build_tests=ON ${GTEST_DIR} +``` Make sure you have Python installed, as some of Google Test's tests are written in Python. If the cmake command complains about not being @@ -264,12 +300,16 @@ able to find Python (`Could NOT find PythonInterp (missing: PYTHON_EXECUTABLE)`), try telling it explicitly where your Python executable can be found: - cmake -DPYTHON_EXECUTABLE=path/to/python -Dgtest_build_tests=ON ${GTEST_DIR} +```bash +cmake -DPYTHON_EXECUTABLE=path/to/python -Dgtest_build_tests=ON ${GTEST_DIR} +``` Next, you can build Google Test and all of its own tests. On \*nix, this is usually done by 'make'. To run the tests, do - make test +```bash +make test +``` All tests should pass. diff --git a/tests/hash-target.cpp b/tests/hash-target.cpp index 1e988c302..e95475cac 100644 --- a/tests/hash-target.cpp +++ b/tests/hash-target.cpp @@ -61,7 +61,7 @@ int main(int argc, char *argv[]) { for (int i = 31; i >= 0; i--) { val = val * 256 + 255; ((char *) &h)[i] = static_cast<char>(static_cast<uint64_t>(val / diff)); - val %= diff.convert_to<uint64_t>(); + val %= (diff & 0xffffffffffffffff).convert_to<uint64_t>(); } if (check_hash(h, diff) != true) { return 3; diff --git a/tests/performance_tests/check_hash.h b/tests/performance_tests/check_hash.h index d24001903..53746fec4 100644 --- a/tests/performance_tests/check_hash.h +++ b/tests/performance_tests/check_hash.h @@ -44,13 +44,13 @@ public: difficulty = difficulty_high; difficulty = (difficulty << 64) | difficulty_low; boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; - ((uint64_t*)&hash)[0] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + ((uint64_t*)&hash)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); hash_value >>= 64; - ((uint64_t*)&hash)[1] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + ((uint64_t*)&hash)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); hash_value >>= 64; - ((uint64_t*)&hash)[2] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + ((uint64_t*)&hash)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); hash_value >>= 64; - ((uint64_t*)&hash)[3] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + ((uint64_t*)&hash)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); return true; } diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 56a1f8c4d..1c4c4384c 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -72,6 +72,7 @@ set(unit_tests_sources parse_amount.cpp pruning.cpp random.cpp + rolling_median.cpp serialization.cpp sha256.cpp slow_memmem.cpp diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 0b267172f..17fba90c6 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -36,6 +36,7 @@ #include "cryptonote_protocol/cryptonote_protocol_handler.inl" #define MAKE_IPV4_ADDRESS(a,b,c,d) epee::net_utils::ipv4_network_address{MAKE_IP(a,b,c,d),0} +#define MAKE_IPV4_SUBNET(a,b,c,d,e) epee::net_utils::ipv4_network_subnet{MAKE_IP(a,b,c,d),e} namespace cryptonote { class blockchain_storage; @@ -93,11 +94,10 @@ typedef nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<test_cor static bool is_blocked(Server &server, const epee::net_utils::network_address &address, time_t *t = NULL) { - const std::string host = address.host_str(); - std::map<std::string, time_t> hosts = server.get_blocked_hosts(); + std::map<epee::net_utils::network_address, time_t> hosts = server.get_blocked_hosts(); for (auto rec: hosts) { - if (rec.first == host) + if (rec.first == address) { if (t) *t = rec.second; @@ -208,5 +208,37 @@ TEST(ban, limit) ASSERT_TRUE(is_blocked(server,MAKE_IPV4_ADDRESS(1,2,3,4))); } +TEST(ban, subnet) +{ + time_t seconds; + test_core pr_core; + cryptonote::t_cryptonote_protocol_handler<test_core> cprotocol(pr_core, NULL); + Server server(cprotocol); + cprotocol.set_p2p_endpoint(&server); + + ASSERT_TRUE(server.block_subnet(MAKE_IPV4_SUBNET(1,2,3,4,24), 10)); + ASSERT_TRUE(server.get_blocked_subnets().size() == 1); + ASSERT_TRUE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,3,4), &seconds)); + ASSERT_TRUE(seconds >= 9); + ASSERT_TRUE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,3,255), &seconds)); + ASSERT_TRUE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,3,0), &seconds)); + ASSERT_FALSE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,4,0), &seconds)); + ASSERT_FALSE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,2,0), &seconds)); + ASSERT_TRUE(server.unblock_subnet(MAKE_IPV4_SUBNET(1,2,3,8,24))); + ASSERT_TRUE(server.get_blocked_subnets().size() == 0); + ASSERT_FALSE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,3,255), &seconds)); + ASSERT_FALSE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,2,3,0), &seconds)); + ASSERT_TRUE(server.block_subnet(MAKE_IPV4_SUBNET(1,2,3,4,8), 10)); + ASSERT_TRUE(server.get_blocked_subnets().size() == 1); + ASSERT_TRUE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,255,3,255), &seconds)); + ASSERT_TRUE(server.is_host_blocked(MAKE_IPV4_ADDRESS(1,0,3,255), &seconds)); + ASSERT_FALSE(server.unblock_subnet(MAKE_IPV4_SUBNET(1,2,3,8,24))); + ASSERT_TRUE(server.get_blocked_subnets().size() == 1); + ASSERT_TRUE(server.block_subnet(MAKE_IPV4_SUBNET(1,2,3,4,8), 10)); + ASSERT_TRUE(server.get_blocked_subnets().size() == 1); + ASSERT_TRUE(server.unblock_subnet(MAKE_IPV4_SUBNET(1,255,0,0,8))); + ASSERT_TRUE(server.get_blocked_subnets().size() == 0); +} + namespace nodetool { template class node_server<cryptonote::t_cryptonote_protocol_handler<test_core>>; } namespace cryptonote { template class t_cryptonote_protocol_handler<test_core>; } diff --git a/tests/unit_tests/difficulty.cpp b/tests/unit_tests/difficulty.cpp index 090fecc84..a732e6969 100644 --- a/tests/unit_tests/difficulty.cpp +++ b/tests/unit_tests/difficulty.cpp @@ -42,13 +42,13 @@ static crypto::hash MKHASH(uint64_t high, uint64_t low) hash_target = (hash_target << 64) | low; boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; crypto::hash h; - ((uint64_t*)&h)[0] = hash_value.convert_to<uint64_t>(); + ((uint64_t*)&h)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); hash_value >>= 64; - ((uint64_t*)&h)[1] = hash_value.convert_to<uint64_t>(); + ((uint64_t*)&h)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); hash_value >>= 64; - ((uint64_t*)&h)[2] = hash_value.convert_to<uint64_t>(); + ((uint64_t*)&h)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); hash_value >>= 64; - ((uint64_t*)&h)[3] = hash_value.convert_to<uint64_t>(); + ((uint64_t*)&h)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); return h; } diff --git a/tests/unit_tests/keccak.cpp b/tests/unit_tests/keccak.cpp index 37da65d76..f4d41a8fa 100644 --- a/tests/unit_tests/keccak.cpp +++ b/tests/unit_tests/keccak.cpp @@ -148,3 +148,20 @@ TEST(keccak, 137_and_1_136) TEST_KECCAK(137, chunks); } +TEST(keccak, alignment) +{ + uint8_t data[6064]; + __attribute__ ((aligned(16))) char adata[6000]; + + for (size_t i = 0; i < sizeof(data) / sizeof(data[0]); ++i) + data[i] = i & 1; + + uint8_t md[32], amd[32]; + for (int offset = 0; offset < 64; ++offset) + { + memcpy(adata, data + offset, 6000); + keccak((const uint8_t*)&data[offset], 6000, md, 32); + keccak((const uint8_t*)adata, 6000, amd, 32); + ASSERT_TRUE(!memcmp(md, amd, 32)); + } +} diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index 326e63db8..3acf75f3b 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -524,6 +524,24 @@ TEST(get_network_address, ipv4) EXPECT_STREQ("23.0.0.254:2000", address->str().c_str()); } +TEST(get_network_address, ipv4subnet) +{ + expect<epee::net_utils::ipv4_network_subnet> address = net::get_ipv4_subnet_address("0.0.0.0", true); + EXPECT_STREQ("0.0.0.0/32", address->str().c_str()); + + address = net::get_ipv4_subnet_address("0.0.0.0"); + EXPECT_TRUE(!address); + + address = net::get_ipv4_subnet_address("0.0.0.0/32"); + EXPECT_STREQ("0.0.0.0/32", address->str().c_str()); + + address = net::get_ipv4_subnet_address("0.0.0.0/0"); + EXPECT_STREQ("0.0.0.0/0", address->str().c_str()); + + address = net::get_ipv4_subnet_address("12.34.56.78/16"); + EXPECT_STREQ("12.34.0.0/16", address->str().c_str()); +} + namespace { using stream_type = boost::asio::ip::tcp; diff --git a/tests/unit_tests/output_distribution.cpp b/tests/unit_tests/output_distribution.cpp index 2f3c78642..38f442c59 100644 --- a/tests/unit_tests/output_distribution.cpp +++ b/tests/unit_tests/output_distribution.cpp @@ -91,36 +91,43 @@ bool get_output_distribution(uint64_t amount, uint64_t from, uint64_t to, uint64 return r && bc->get_output_distribution(amount, from, to, start_height, distribution, base); } +crypto::hash get_block_hash(uint64_t height) +{ + crypto::hash hash; + *((uint64_t*)&hash) = height; + return hash; +} + TEST(output_distribution, extend) { boost::optional<cryptonote::rpc::output_distribution_data> res; - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 29, false); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 29, ::get_block_hash, false, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 2); ASSERT_EQ(res->distribution, std::vector<uint64_t>({5, 0})); - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 29, true); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 29, ::get_block_hash, true, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 2); ASSERT_EQ(res->distribution, std::vector<uint64_t>({55, 55})); - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 30, false); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 30, ::get_block_hash, false, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 3); ASSERT_EQ(res->distribution, std::vector<uint64_t>({5, 0, 2})); - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 30, true); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 30, ::get_block_hash, true, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 3); ASSERT_EQ(res->distribution, std::vector<uint64_t>({55, 55, 57})); - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 31, false); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 31, ::get_block_hash, false, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 4); ASSERT_EQ(res->distribution, std::vector<uint64_t>({5, 0, 2, 3})); - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 31, true); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 28, 31, ::get_block_hash, true, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 4); ASSERT_EQ(res->distribution, std::vector<uint64_t>({55, 55, 57, 60})); @@ -130,7 +137,7 @@ TEST(output_distribution, one) { boost::optional<cryptonote::rpc::output_distribution_data> res; - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 0, 0, false); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 0, 0, ::get_block_hash, false, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 1); ASSERT_EQ(res->distribution.back(), 0); @@ -140,7 +147,7 @@ TEST(output_distribution, full_cumulative) { boost::optional<cryptonote::rpc::output_distribution_data> res; - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 0, 31, true); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 0, 31, ::get_block_hash, true, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 32); ASSERT_EQ(res->distribution.back(), 60); @@ -150,7 +157,7 @@ TEST(output_distribution, full_noncumulative) { boost::optional<cryptonote::rpc::output_distribution_data> res; - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 0, 31, false); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 0, 31, ::get_block_hash, false, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 32); for (size_t i = 0; i < 32; ++i) @@ -161,7 +168,7 @@ TEST(output_distribution, part_cumulative) { boost::optional<cryptonote::rpc::output_distribution_data> res; - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 4, 8, true); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 4, 8, ::get_block_hash, true, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 5); ASSERT_EQ(res->distribution, std::vector<uint64_t>({0, 1, 6, 7, 11})); @@ -171,7 +178,7 @@ TEST(output_distribution, part_noncumulative) { boost::optional<cryptonote::rpc::output_distribution_data> res; - res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 4, 8, false); + res = cryptonote::rpc::RpcHandler::get_output_distribution(::get_output_distribution, 0, 4, 8, ::get_block_hash, false, test_distribution_size); ASSERT_TRUE(res != boost::none); ASSERT_EQ(res->distribution.size(), 5); ASSERT_EQ(res->distribution, std::vector<uint64_t>({0, 1, 5, 1, 4})); diff --git a/tests/unit_tests/output_selection.cpp b/tests/unit_tests/output_selection.cpp index 0094fc765..235b1c809 100644 --- a/tests/unit_tests/output_selection.cpp +++ b/tests/unit_tests/output_selection.cpp @@ -172,7 +172,7 @@ TEST(select_outputs, density) float chain_ratio = count_chain / (float)n_outs; MDEBUG(count_selected << "/" << NPICKS << " outputs selected in blocks of density " << d << ", " << 100.0f * selected_ratio << "%"); MDEBUG(count_chain << "/" << offsets.size() << " outputs in blocks of density " << d << ", " << 100.0f * chain_ratio << "%"); - ASSERT_LT(fabsf(selected_ratio - chain_ratio), 0.02f); + ASSERT_LT(fabsf(selected_ratio - chain_ratio), 0.025f); } } diff --git a/tests/unit_tests/rolling_median.cpp b/tests/unit_tests/rolling_median.cpp new file mode 100644 index 000000000..6d6adcc7d --- /dev/null +++ b/tests/unit_tests/rolling_median.cpp @@ -0,0 +1,202 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <random> +#include "gtest/gtest.h" +#include "misc_language.h" +#include "rolling_median.h" +#include "crypto/crypto.h" + +TEST(rolling_median, one) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(1); + m.insert(42); + ASSERT_EQ(m.median(), 42); + m.insert(18); + ASSERT_EQ(m.median(), 18); + m.insert(7483); + ASSERT_EQ(m.median(), 7483); +} + +TEST(rolling_median, two) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(2); + m.insert(42); + ASSERT_EQ(m.median(), 42); + m.insert(45); + ASSERT_EQ(m.median(), 43); + m.insert(49); + ASSERT_EQ(m.median(), 47); + m.insert(41); + ASSERT_EQ(m.median(), 45); + m.insert(43); + ASSERT_EQ(m.median(), 42); + m.insert(40); + ASSERT_EQ(m.median(), 41); + m.insert(41); + ASSERT_EQ(m.median(), 40); +} + +TEST(rolling_median, series) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(100); + std::vector<uint64_t> v; + v.reserve(100); + for (int i = 0; i < 10000; ++i) + { + uint64_t r = rand(); + v.push_back(r); + if (v.size() > 100) + v.erase(v.begin()); + m.insert(r); + std::vector<uint64_t> vcopy = v; + ASSERT_EQ(m.median(), epee::misc_utils::median(vcopy)); + } +} + +TEST(rolling_median, clear_whole) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(100); + std::vector<uint64_t> random, median; + random.reserve(10000); + median.reserve(10000); + for (int i = 0; i < 10000; ++i) + { + random.push_back(rand()); + m.insert(random.back()); + median.push_back(m.median()); + } + m.clear(); + for (int i = 0; i < 10000; ++i) + { + m.insert(random[i]); + ASSERT_EQ(median[i], m.median()); + } +} + +TEST(rolling_median, clear_partway) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(100); + std::vector<uint64_t> random, median; + random.reserve(10000); + median.reserve(10000); + for (int i = 0; i < 10000; ++i) + { + random.push_back(rand()); + m.insert(random.back()); + median.push_back(m.median()); + } + m.clear(); + for (int i = 10000 - 100; i < 10000; ++i) + { + m.insert(random[i]); + } + ASSERT_EQ(median[10000-1], m.median()); +} + +TEST(rolling_median, order) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(1000); + std::vector<uint64_t> random; + random.reserve(1000); + for (int i = 0; i < 1000; ++i) + { + random.push_back(rand()); + m.insert(random.back()); + } + const uint64_t med = m.median(); + + std::sort(random.begin(), random.end(), [](uint64_t a, uint64_t b) { return a < b; }); + m.clear(); + for (int i = 0; i < 1000; ++i) + m.insert(random[i]); + ASSERT_EQ(med, m.median()); + + std::sort(random.begin(), random.end(), [](uint64_t a, uint64_t b) { return a > b; }); + m.clear(); + for (int i = 0; i < 1000; ++i) + m.insert(random[i]); + ASSERT_EQ(med, m.median()); + + std::shuffle(random.begin(), random.end(), std::default_random_engine(crypto::rand<unsigned>())); + m.clear(); + for (int i = 0; i < 1000; ++i) + m.insert(random[i]); + ASSERT_EQ(med, m.median()); +} + +TEST(rolling_median, history_blind) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(10); + + uint64_t median = 0; + for (int i = 0; i < 1000; ++i) + { + m.clear(); + int history_length = 743723 % (i+1); + while (history_length--) + m.insert(743284 % (i+1)); + for (int j = 0; j < 10; ++j) + m.insert(8924829384 % (j+1)); + if (i == 0) + median = m.median(); + else + ASSERT_EQ(median, m.median()); + } +} + +TEST(rolling_median, size) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(10); + + ASSERT_EQ(m.size(), 0); + m.insert(1); + ASSERT_EQ(m.size(), 1); + m.insert(2); + ASSERT_EQ(m.size(), 2); + m.clear(); + ASSERT_EQ(m.size(), 0); + for (int i = 0; i < 10; ++i) + { + m.insert(80 % (i + 1)); + ASSERT_EQ(m.size(), i + 1); + } + m.insert(1); + ASSERT_EQ(m.size(), 10); + m.insert(2); + ASSERT_EQ(m.size(), 10); + m.clear(); + ASSERT_EQ(m.size(), 0); + m.insert(4); + ASSERT_EQ(m.size(), 1); + for (int i = 0; i < 1000; ++i) + { + m.insert(80 % (i + 1)); + ASSERT_EQ(m.size(), std::min<int>(10, i + 2)); + } +} diff --git a/tests/unit_tests/test_protocol_pack.cpp b/tests/unit_tests/test_protocol_pack.cpp index 7329c0d23..0ae2e9c68 100644 --- a/tests/unit_tests/test_protocol_pack.cpp +++ b/tests/unit_tests/test_protocol_pack.cpp @@ -48,6 +48,7 @@ TEST(protocol_pack, protocol_pack_command) cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request r2; res = epee::serialization::load_t_from_binary(r2, buff); + ASSERT_TRUE(res); ASSERT_TRUE(r.m_block_ids.size() == i); ASSERT_TRUE(r.start_height == 1); ASSERT_TRUE(r.total_height == 3); diff --git a/tests/unit_tests/uri.cpp b/tests/unit_tests/uri.cpp index df1dbc130..04d935751 100644 --- a/tests/unit_tests/uri.cpp +++ b/tests/unit_tests/uri.cpp @@ -144,9 +144,7 @@ TEST(uri, bad_payment_id) TEST(uri, short_payment_id) { - PARSE_URI("monero:" TEST_ADDRESS"?tx_payment_id=1234567890123456", true); - ASSERT_EQ(address, TEST_ADDRESS); - ASSERT_EQ(payment_id, "1234567890123456"); + PARSE_URI("monero:" TEST_ADDRESS"?tx_payment_id=1234567890123456", false); } TEST(uri, long_payment_id) diff --git a/tests/unit_tests/varint.cpp b/tests/unit_tests/varint.cpp index ca0900682..72691d722 100644 --- a/tests/unit_tests/varint.cpp +++ b/tests/unit_tests/varint.cpp @@ -56,7 +56,6 @@ TEST(varint, equal) ASSERT_TRUE (bytes > 0 && bytes <= sizeof(buf)); uint64_t idx2; - bufptr = buf; std::string s(buf, bytes); int read = tools::read_varint(s.begin(), s.end(), idx2); ASSERT_EQ (read, bytes); diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 04fc5b5cf..18ce37221 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -67,11 +67,13 @@ class Daemon(object): } return self.rpc.send_json_rpc_request(submitblock) - def getblock(self, height=0): + def getblock(self, hash = '', height = 0, fill_pow_hash = False): getblock = { 'method': 'getblock', 'params': { - 'height': height + 'hash': hash, + 'height': height, + 'fill_pow_hash': fill_pow_hash, }, 'jsonrpc': '2.0', 'id': '0' |