diff options
138 files changed, 3943 insertions, 1166 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..50297e146 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,62 @@ +name: continuous-integration/gh-actions/cli + +on: [push, pull_request] + +jobs: + build-macos: + runs-on: macOS-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: update brew and install dependencies + run: brew update && brew install boost hidapi zmq libpgm unbound libsodium miniupnpc ldns expat libunwind-headers protobuf + - name: build + run: make -j3 + + build-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - uses: numworks/setup-msys2@v1 + - name: update pacman + run: msys2do pacman -Syu --noconfirm + - name: install monero dependencies + run: msys2do pacman -S --noconfirm mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git + - name: build + run: msys2do make release-static-win64 -j3 + + build-ubuntu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: remove bundled boost + run: sudo rm -rf /usr/local/share/boost + - name: update apt + run: sudo apt update + - name: install monero dependencies + run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev + - name: build + run: make -j3 + + test-ubuntu: + needs: build-ubuntu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: remove bundled boost + run: sudo rm -rf /usr/local/share/boost + - name: update apt + run: sudo apt update + - name: install monero dependencies + run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev + - name: install requests + run: pip install requests + - name: tests + run: make release-test -j3 diff --git a/.gitignore b/.gitignore index 0ece7cb75..08c310e66 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,5 @@ nbproject .idea/ /testnet + +__pycache__/ diff --git a/ANONYMITY_NETWORKS.md b/ANONYMITY_NETWORKS.md index cb0e378c3..3337b5fc3 100644 --- a/ANONYMITY_NETWORKS.md +++ b/ANONYMITY_NETWORKS.md @@ -46,8 +46,10 @@ Connecting to an anonymous address requires the command line option `--tx-proxy` which tells `monerod` the ip/port of a socks proxy provided by a separate process. On most systems the configuration will look like: -> `--tx-proxy tor,127.0.0.1:9050,10` -> `--tx-proxy i2p,127.0.0.1:9000` +``` +--tx-proxy tor,127.0.0.1:9050,10 +--tx-proxy i2p,127.0.0.1:9000 +``` which tells `monerod` that ".onion" p2p addresses can be forwarded to a socks proxy at IP 127.0.0.1 port 9050 with a max of 10 outgoing connections and @@ -55,12 +57,14 @@ proxy at IP 127.0.0.1 port 9050 with a max of 10 outgoing connections and 9000 with the default max outgoing connections. Since there are no seed nodes for anonymity connections, peers must be manually specified: -> `--add-exclusive-node rveahdfho7wo4b2m.onion:28083` -> `--add-peer rveahdfho7wo4b2m.onion:28083` +``` +--add-exclusive-node rveahdfho7wo4b2m.onion:28083 +--add-peer rveahdfho7wo4b2m.onion:28083 +``` Either option can be listed multiple times, and can specify any mix of Tor, I2P, and IPv4 addresses. Using `--add-exclusive-node` will prevent the usage of -seed nodes on ALL networks, which will typically be undesireable. +seed nodes on ALL networks, which will typically be undesirable. ### Inbound Connections @@ -68,8 +72,10 @@ Receiving anonymity connections is done through the option `--anonymous-inbound`. This option tells `monerod` the inbound address, network type, and max connections: -> `--anonymous-inbound rveahdfho7wo4b2m.onion:28083,127.0.0.1:28083,25` -> `--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000,127.0.0.1:30000` +``` +--anonymous-inbound rveahdfho7wo4b2m.onion:28083,127.0.0.1:28083,25 +--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000,127.0.0.1:30000 +``` which tells `monerod` that a max of 25 inbound Tor connections are being received at address "rveahdfho7wo4b2m.onion:28083" and forwarded to `monerod` @@ -87,12 +93,16 @@ P2P anonymity connections. The anonymity network (Tor/i2p) is [configured in the same manner](#configuration), except the localhost port must be the RPC port (typically 18081 for mainnet) instead of the p2p port: -> HiddenServiceDir /var/lib/tor/data/monero -> HiddenServicePort 18081 127.0.0.1:18081 +``` +HiddenServiceDir /var/lib/tor/data/monero +HiddenServicePort 18081 127.0.0.1:18081 +``` Then the wallet will be configured to use a Tor/i2p address: -> `--proxy 127.0.0.1:9050` -> `--daemon-address rveahdfho7wo4b2m.onion` +``` +--proxy 127.0.0.1:9050 +--daemon-address rveahdfho7wo4b2m.onion +``` The proxy must match the address type - a Tor proxy will not work properly with i2p addresses, etc. @@ -125,8 +135,10 @@ can distribute the address to its other peers. Tor must be configured for hidden services. An example configuration ("torrc") might look like: -> HiddenServiceDir /var/lib/tor/data/monero -> HiddenServicePort 28083 127.0.0.1:28083 +``` +HiddenServiceDir /var/lib/tor/data/monero +HiddenServicePort 28083 127.0.0.1:28083 +``` This will store key information in `/var/lib/tor/data/monero` and will forward "Tor port" 28083 to port 28083 of ip 127.0.0.1. The file @@ -170,7 +182,7 @@ be used by an ISP to link a user to a transaction. Run `monerod` as often as possible to conceal when transactions are being sent. Future versions will also have peers that first receive a transaction over an anonymity network delay the broadcast to public peers by a randomized amount. -This will not completetely mitigate a user who syncs up sends then quits, in +This will not completely mitigate a user who syncs up sends then quits, in part because this rule is not enforceable, so this mitigation strategy is simply a best effort attempt. @@ -183,9 +195,9 @@ the connections are not circuit based. #### Mitigation -The best mitigiation is to use I2P instead of Tor. However, I2P +The best mitigation is to use I2P instead of Tor. However, I2P has a smaller set of users (less cover traffic) and academic reviews, so there -is a tradeoff in potential isses. Also, anyone attempting this strategy really +is a trade off in potential issues. Also, anyone attempting this strategy really wants to uncover a user, it seems unlikely that this would be performed against every Tor/I2P user. @@ -213,7 +225,7 @@ key identity. @secparam (twitter) recommended changing circuits (Tor) as an additional precaution. This is likely not a good idea - forcibly requesting Tor to change circuits is observable by the ISP. Instead, `monerod` should likely disconnect -from peers ocassionally. Tor will rotate circuits every ~10 minutes, so +from peers occasionally. Tor will rotate circuits every ~10 minutes, so establishing new connections will use a new public key identity and make it more difficult for the hidden service to link information. This process will have to be done carefully because closing/reconnecting connections can also @@ -81,11 +81,11 @@ debug-static-all: debug-static-win64: mkdir -p $(builddir)/debug - cd $(builddir)/debug && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Debug -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 $(topdir) && $(MAKE) + cd $(builddir)/debug && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Debug -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=$(shell cd ${MINGW_PREFIX}/.. && pwd -W) $(topdir) && $(MAKE) debug-static-win32: mkdir -p $(builddir)/debug - cd $(builddir)/debug && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Debug -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 $(topdir) && $(MAKE) + cd $(builddir)/debug && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Debug -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=$(shell cd ${MINGW_PREFIX}/.. && pwd -W) $(topdir) && $(MAKE) cmake-release: mkdir -p $(builddir)/release @@ -152,11 +152,11 @@ release-static-linux-i686: release-static-win64: mkdir -p $(builddir)/release - cd $(builddir)/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 $(topdir) && $(MAKE) + cd $(builddir)/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=$(shell cd ${MINGW_PREFIX}/.. && pwd -W) $(topdir) && $(MAKE) release-static-win32: mkdir -p $(builddir)/release - cd $(builddir)/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 $(topdir) && $(MAKE) + cd $(builddir)/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=$(topdir)/cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=$(shell cd ${MINGW_PREFIX}/.. && pwd -W) $(topdir) && $(MAKE) fuzz: mkdir -p $(builddir)/fuzz @@ -54,31 +54,11 @@ Our researchers are available on IRC in [#monero-research-lab on Freenode](https - You can subscribe to an [announcement listserv](https://lists.getmonero.org) to get critical announcements from the Monero core team. The announcement list can be very helpful for knowing when software updates are needed. ## Translations -The CLI wallet is available in different languages. If you want to help translate it, see our self-hosted localization platform, Weblate, on [translate.getmonero.org](https://translate.getmonero.org/projects/CLI/). Every translation *must* be uploaded on the platform, pull requests directly editing the code in this repository will be closed. If you need help with Weblate, you can find a guide with screenshots [here](https://github.com/monero-ecosystem/monero-translations/blob/master/weblate.md). +The CLI wallet is available in different languages. If you want to help translate it, see our self-hosted localization platform, Weblate, on [translate.getmonero.org]( https://translate.getmonero.org/projects/monero/cli-wallet/). Every translation *must* be uploaded on the platform, pull requests directly editing the code in this repository will be closed. If you need help with Weblate, you can find a guide with screenshots [here](https://github.com/monero-ecosystem/monero-translations/blob/master/weblate.md). If you need help/support/info about translations, contact the localization workgroup. You can find the complete list of contacts on the repository of the workgroup: [monero-translations](https://github.com/monero-ecosystem/monero-translations#contacts). -## Build Status - -### IMPORTANT - -These builds are of the master branch, which is used for active development and can be either unstable or incompatible with release software. Please compile release branches. - -| Operating System | Processor | Status | -| --------------------- | -------- |--------| -| Ubuntu 16.04 | i686 | [](https://build.getmonero.org/builders/monero-static-ubuntu-i686) -| Ubuntu 16.04 | amd64 | [](https://build.getmonero.org/builders/monero-static-ubuntu-amd64) -| Ubuntu 16.04 | armv7 | [](https://build.getmonero.org/builders/monero-static-ubuntu-arm7) -| Debian Stable | armv8 | [](https://build.getmonero.org/builders/monero-static-debian-armv8) -| macOS 10.11 | amd64 | [](https://build.getmonero.org/builders/monero-static-osx-10.11) -| macOS 10.12 | amd64 | [](https://build.getmonero.org/builders/monero-static-osx-10.12) -| macOS 10.13 | amd64 | [](https://build.getmonero.org/builders/monero-static-osx-10.13) -| FreeBSD 11 | amd64 | [](https://build.getmonero.org/builders/monero-static-freebsd64) -| DragonFly BSD 4.6 | amd64 | [](https://build.getmonero.org/builders/monero-static-dragonflybsd-amd64) -| Windows (MSYS2/MinGW) | i686 | [](https://build.getmonero.org/builders/monero-static-win32) -| Windows (MSYS2/MinGW) | amd64 | [](https://build.getmonero.org/builders/monero-static-win64) - ## Coverage | Type | Status | @@ -157,7 +137,8 @@ Dates are provided in the format YYYY-MM-DD. | XXXXXXX | XXX-XX-XX | XXX | vX.XX.X.X | vX.XX.X.X | XXX | X's indicate that these details have not been determined as of commit date. -* indicates estimate as of commit date + +\* indicates estimate as of commit date ## Release staging schedule and protocol diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index 7eedee15e..28ec972e4 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -58,6 +58,7 @@ ifeq ($(host_os),) host_os:=$(findstring linux,$(full_host_os)) endif host_os+=$(findstring darwin,$(full_host_os)) +host_os+=$(findstring freebsd,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) host_os:=$(strip $(host_os)) ifeq ($(host_os),) @@ -74,6 +75,9 @@ endif ifeq ($(host_os),linux) host_cmake=Linux endif +ifeq ($(host_os),freebsd) +host_cmake=FreeBSD +endif ifeq ($(host_os),darwin) host_cmake=Darwin endif diff --git a/contrib/depends/builders/default.mk b/contrib/depends/builders/default.mk index f097db65d..c4191435d 100644 --- a/contrib/depends/builders/default.mk +++ b/contrib/depends/builders/default.mk @@ -17,4 +17,4 @@ define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach flags, CFLAGS CXXFLAGS LDFLAGS, $(eval $(call add_build_flags_func,$(flags)))) +$(foreach flags, CFLAGS CXXFLAGS ARFLAGS LDFLAGS, $(eval $(call add_build_flags_func,$(flags)))) diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index 469144361..355ae07eb 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -10,6 +10,7 @@ $(1)_libtool=$($($(1)_type)_LIBTOOL) $(1)_nm=$($($(1)_type)_NM) $(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) $(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) +$(1)_arflags=$($($(1)_type)_ARFLAGS) $($($(1)_type)_$(release_type)_ARFLAGS) $(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib $(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include $(1)_recipe_hash:= @@ -102,6 +103,11 @@ $(1)_cxxflags+=$($(1)_cxxflags_$(host_arch)) $($(1)_cxxflags_$(host_arch)_$(rele $(1)_cxxflags+=$($(1)_cxxflags_$(host_os)) $($(1)_cxxflags_$(host_os)_$(release_type)) $(1)_cxxflags+=$($(1)_cxxflags_$(host_arch)_$(host_os)) $($(1)_cxxflags_$(host_arch)_$(host_os)_$(release_type)) +$(1)_arflags+=$($(1)_arflags_$(release_type)) +$(1)_arflags+=$($(1)_arflags_$(host_arch)) $($(1)_arflags_$(host_arch)_$(release_type)) +$(1)_arflags+=$($(1)_arflags_$(host_os)) $($(1)_arflags_$(host_os)_$(release_type)) +$(1)_arflags+=$($(1)_arflags_$(host_arch)_$(host_os)) $($(1)_arflags_$(host_arch)_$(host_os)_$(release_type)) + $(1)_cppflags+=$($(1)_cppflags_$(release_type)) $(1)_cppflags+=$($(1)_cppflags_$(host_arch)) $($(1)_cppflags_$(host_arch)_$(release_type)) $(1)_cppflags+=$($(1)_cppflags_$(host_os)) $($(1)_cppflags_$(host_os)_$(release_type)) @@ -143,6 +149,9 @@ endif ifneq ($($(1)_ar),) $(1)_autoconf += AR="$$($(1)_ar)" endif +ifneq ($($(1)_arflags),) +$(1)_autoconf += ARFLAGS="$$($(1)_arflags)" +endif ifneq ($($(1)_cflags),) $(1)_autoconf += CFLAGS="$$($(1)_cflags)" endif diff --git a/contrib/depends/hosts/android.mk b/contrib/depends/hosts/android.mk index 2a24435d4..d6f8b99dd 100644 --- a/contrib/depends/hosts/android.mk +++ b/contrib/depends/hosts/android.mk @@ -6,9 +6,11 @@ endif android_CC=$(host_toolchain)clang android_CXX=$(host_toolchain)clang++ +android_RANLIB=: android_CFLAGS=-pipe android_CXXFLAGS=$(android_CFLAGS) +android_ARFLAGS=crsD android_release_CFLAGS=-O2 android_release_CXXFLAGS=$(android_release_CFLAGS) diff --git a/contrib/depends/hosts/darwin.mk b/contrib/depends/hosts/darwin.mk index 6f8f2ffe2..7b5c8b051 100644 --- a/contrib/depends/hosts/darwin.mk +++ b/contrib/depends/hosts/darwin.mk @@ -7,6 +7,7 @@ darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sys darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) +darwin_ARFLAGS=cr darwin_release_CFLAGS=-O1 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) diff --git a/contrib/depends/hosts/default.mk b/contrib/depends/hosts/default.mk index 6f60d6b3f..2e7f9fa23 100644 --- a/contrib/depends/hosts/default.mk +++ b/contrib/depends/hosts/default.mk @@ -23,4 +23,4 @@ host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef $(foreach tool,CC CXX AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL,$(eval $(call add_host_tool_func,$(tool)))) -$(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) +$(foreach flags,CFLAGS CXXFLAGS ARFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/contrib/depends/hosts/freebsd.mk b/contrib/depends/hosts/freebsd.mk new file mode 100644 index 000000000..2e3b5933e --- /dev/null +++ b/contrib/depends/hosts/freebsd.mk @@ -0,0 +1,18 @@ +freebsd_CC=clang-8 +freebsd_CXX=clang++-8 +freebsd_AR=ar +freebsd_RANLIB=ranlib +freebsd_NM=nm + +freebsd_CFLAGS=-pipe +freebsd_CXXFLAGS=$(freebsd_CFLAGS) +freebsd_ARFLAGS=cr + +freebsd_release_CFLAGS=-O2 +freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS) + +freebsd_debug_CFLAGS=-g -O0 +freebsd_debug_CXXFLAGS=$(freebsd_debug_CFLAGS) + +freebsd_native_toolchain=freebsd_base + diff --git a/contrib/depends/hosts/linux.mk b/contrib/depends/hosts/linux.mk index b13a0f1ad..912fdb03c 100644 --- a/contrib/depends/hosts/linux.mk +++ b/contrib/depends/hosts/linux.mk @@ -1,5 +1,6 @@ linux_CFLAGS=-pipe linux_CXXFLAGS=$(linux_CFLAGS) +linux_ARFLAGS=cr linux_release_CFLAGS=-O2 linux_release_CXXFLAGS=$(linux_release_CFLAGS) diff --git a/contrib/depends/hosts/mingw32.mk b/contrib/depends/hosts/mingw32.mk index dbfb62fdc..ccc4c5082 100644 --- a/contrib/depends/hosts/mingw32.mk +++ b/contrib/depends/hosts/mingw32.mk @@ -1,5 +1,6 @@ mingw32_CFLAGS=-pipe mingw32_CXXFLAGS=$(mingw32_CFLAGS) +mingw32_ARFLAGS=cr mingw32_release_CFLAGS=-O2 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) diff --git a/contrib/depends/packages/boost.mk b/contrib/depends/packages/boost.mk index 3231df925..0d241928e 100644 --- a/contrib/depends/packages/boost.mk +++ b/contrib/depends/packages/boost.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://dl.bintray.com/boostorg/release/1.64.0/source/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 $(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 $(package)_dependencies=libiconv +$(package)_patches=fix_aroptions.patch define $(package)_set_vars $(package)_config_opts_release=variant=release @@ -24,10 +25,12 @@ $(package)_archiver_darwin=$($(package)_libtool) $(package)_config_libraries=chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale $(package)_cxxflags=-std=c++11 $(package)_cxxflags_linux=-fPIC +$(package)_cxxflags_freebsd=-fPIC endef define $(package)_preprocess_cmds - echo "using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$(boost_archiver_$(host_os))\" <striper>\"$(host_STR IP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam + patch -p1 < $($(package)_patch_dir)/fix_aroptions.patch &&\ + echo "using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$(boost_archiver_$(host_os))\" <arflags>\"$($(package)_arflags)\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam endef define $(package)_config_cmds diff --git a/contrib/depends/packages/eudev.mk b/contrib/depends/packages/eudev.mk index a7795b777..0e930df93 100644 --- a/contrib/depends/packages/eudev.mk +++ b/contrib/depends/packages/eudev.mk @@ -9,7 +9,7 @@ define $(package)_set_vars endef define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmd diff --git a/contrib/depends/packages/freebsd_base.mk b/contrib/depends/packages/freebsd_base.mk new file mode 100644 index 000000000..c6a209dcd --- /dev/null +++ b/contrib/depends/packages/freebsd_base.mk @@ -0,0 +1,23 @@ +package=freebsd_base +$(package)_version=11.3 +$(package)_download_path=https://download.freebsd.org/ftp/releases/amd64/$($(package)_version)-RELEASE/ +$(package)_download_file=base.txz +$(package)_file_name=freebsd-base-$($(package)_version).txz +$(package)_sha256_hash=4599023ac136325b86f2fddeec64c1624daa83657e40b00b2ef944c81463a4ff + +define $(package)_extract_cmds + echo $($(package)_sha256_hash) $($(1)_source_dir)/$($(package)_file_name) | sha256sum -c &&\ + tar xf $($(1)_source_dir)/$($(package)_file_name) ./lib/ ./usr/lib/ ./usr/include/ +endef + +define $(package)_build_cmds + mkdir bin &&\ + echo "exec /usr/bin/clang-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang-8 &&\ + echo "exec /usr/bin/clang++-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang++-8 &&\ + chmod 755 bin/* +endef + +define $(package)_stage_cmds + mkdir $($(package)_staging_dir)/$(host_prefix)/native &&\ + mv bin lib usr $($(package)_staging_dir)/$(host_prefix)/native +endef diff --git a/contrib/depends/packages/hidapi.mk b/contrib/depends/packages/hidapi.mk index a27df04fa..b76ef1548 100644 --- a/contrib/depends/packages/hidapi.mk +++ b/contrib/depends/packages/hidapi.mk @@ -18,7 +18,7 @@ endef define $(package)_config_cmds ./bootstrap &&\ - $($(package)_autoconf) $($(package)_config_opts) + $($(package)_autoconf) $($(package)_config_opts) AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk index d4995c1b7..eac8b4331 100644 --- a/contrib/depends/packages/libiconv.mk +++ b/contrib/depends/packages/libiconv.mk @@ -10,6 +10,7 @@ define $(package)_set_vars $(package)_config_opts=--enable-static $(package)_config_opts=--disable-shared $(package)_config_opts_linux=--with-pic + $(package)_config_opts_freebsd=--with-pic endef define $(package)_preprocess_cmds @@ -18,7 +19,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index d865d2a17..348c410a7 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -19,11 +19,11 @@ ifneq ($(host_os),darwin) define $(package)_config_cmds cp -f $(BASEDIR)/config.guess config.guess &&\ cp -f $(BASEDIR)/config.sub config.sub &&\ - $($(package)_autoconf) + $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef else define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef endif diff --git a/contrib/depends/packages/native_cctools.mk b/contrib/depends/packages/native_cctools.mk index bcfe1af6b..8bf71254f 100644 --- a/contrib/depends/packages/native_cctools.mk +++ b/contrib/depends/packages/native_cctools.mk @@ -10,6 +10,7 @@ $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-lin $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9 $(package)_extra_sources=$($(package)_clang_file_name) +$(package)_patches=skip_otool.patch define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ @@ -37,7 +38,10 @@ $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ endef +# If clang gets updated to a version with a fix for https://reviews.llvm.org/D50559 +# then the patch that skips otool can be removed. define $(package)_preprocess_cmds + patch -p0 < $($(package)_patch_dir)/skip_otool.patch && \ cd $($(package)_build_subdir); ./autogen.sh && \ sed -i.old "/define HAVE_PTHREADS/d" ld64/src/ld/InputFiles.h endef diff --git a/contrib/depends/packages/ncurses.mk b/contrib/depends/packages/ncurses.mk index 1acf1faca..d8fdf351c 100644 --- a/contrib/depends/packages/ncurses.mk +++ b/contrib/depends/packages/ncurses.mk @@ -7,7 +7,7 @@ $(package)_patches=fallback.c define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" - $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" + $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" ARFLAGS=$($(package)_arflags) cf_cv_ar_flags="" $(package)_config_env_darwin=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" $(package)_config_opts=--prefix=$(host_prefix) $(package)_config_opts+=--disable-shared diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index f3b781f28..4b07d08ba 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -3,9 +3,10 @@ $(package)_version=1.0.2r $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6 +$(package)_patches=fix_arflags.patch define $(package)_set_vars -$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" +$(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl $(package)_config_opts+=no-capieng $(package)_config_opts+=no-dso @@ -36,6 +37,7 @@ $(package)_config_opts+=no-zlib $(package)_config_opts+=no-zlib-dynamic $(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags) $(package)_config_opts_linux=-fPIC -Wa,--noexecstack +$(package)_config_opts_freebsd=-fPIC -Wa,--noexecstack $(package)_config_opts_x86_64_linux=linux-x86_64 $(package)_config_opts_i686_linux=linux-generic32 $(package)_config_opts_arm_linux=linux-generic32 @@ -49,12 +51,14 @@ $(package)_config_opts_powerpc_linux=linux-generic32 $(package)_config_opts_x86_64_darwin=darwin64-x86_64-cc $(package)_config_opts_x86_64_mingw32=mingw64 $(package)_config_opts_i686_mingw32=mingw +$(package)_config_opts_x86_64_freebsd=BSD-x86_64 endef define $(package)_preprocess_cmds sed -i.old "/define DATE/d" util/mkbuildinf.pl && \ sed -i.old "s|engines apps test|engines|" Makefile.org && \ - sed -i -e "s/-mandroid //" Configure + sed -i -e "s/-mandroid //" Configure && \ + patch < $($(package)_patch_dir)/fix_arflags.patch endef define $(package)_config_cmds diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 9302d5b4d..95b23a37e 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,18 +1,22 @@ -ifeq ($(host_os),android) packages:=boost openssl zeromq libiconv -else -packages:=boost openssl zeromq expat ldns libiconv hidapi protobuf libusb -endif -native_packages := native_ccache native_protobuf +native_packages := native_ccache + +hardware_packages := hidapi protobuf libusb +hardware_native_packages := native_protobuf android_native_packages = android_ndk android_packages = ncurses readline sodium -darwin_native_packages = native_biplist native_ds_store native_mac_alias -darwin_packages = sodium-darwin ncurses readline +darwin_native_packages = native_biplist native_ds_store native_mac_alias $(hardware_native_packages) +darwin_packages = sodium ncurses readline $(hardware_packages) -linux_packages = eudev ncurses readline sodium +# not really native... +freebsd_native_packages = freebsd_base +freebsd_packages = ncurses readline sodium + +linux_packages = eudev ncurses readline sodium $(hardware_packages) +linux_native_packages = $(hardware_native_packages) qt_packages = qt ifeq ($(build_tests),ON) @@ -23,10 +27,8 @@ ifneq ($(host_arch),riscv64) linux_packages += unwind endif -ifeq ($(host_os),mingw32) -packages += icu4c -packages += sodium -endif +mingw32_packages = icu4c sodium $(hardware_packages) +mingw32_native_packages = $(hardware_native_packages) ifneq ($(build_os),darwin) darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus diff --git a/contrib/depends/packages/protobuf.mk b/contrib/depends/packages/protobuf.mk index 81fa78a3f..ad1098975 100644 --- a/contrib/depends/packages/protobuf.mk +++ b/contrib/depends/packages/protobuf.mk @@ -12,7 +12,7 @@ define $(package)_set_vars endef define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/readline.mk b/contrib/depends/packages/readline.mk index 29ae783a2..b6e6a451a 100644 --- a/contrib/depends/packages/readline.mk +++ b/contrib/depends/packages/readline.mk @@ -7,7 +7,7 @@ $(package)_dependencies=ncurses define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" - $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" LDFLAGS="-L$(host_prefix)/lib" + $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" LDFLAGS="-L$(host_prefix)/lib" ARFLAGS=$($(package)_arflags) $(package)_config_env_darwin=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" $(package)_config_opts+=--prefix=$(host_prefix) $(package)_config_opts+=--exec-prefix=$(host_prefix) diff --git a/contrib/depends/packages/sodium-darwin.mk b/contrib/depends/packages/sodium-darwin.mk deleted file mode 100644 index 9f11a9426..000000000 --- a/contrib/depends/packages/sodium-darwin.mk +++ /dev/null @@ -1,30 +0,0 @@ -package=sodium-darwin -$(package)_version=1.0.16 -$(package)_download_path=https://download.libsodium.org/libsodium/releases/ -$(package)_file_name=libsodium-$($(package)_version).tar.gz -$(package)_sha256_hash=eeadc7e1e1bcef09680fb4837d448fbdf57224978f865ac1c16745868fbd0533 - -define $(package)_set_vars -$(package)_build_opts_darwin=OS=Darwin LIBTOOL="$($(package)_libtool)" -$(package)_config_opts=--enable-static --disable-shared -$(package)_config_opts+=--prefix=$(host_prefix) -endef - -define $(package)_config_cmds - ./autogen.sh &&\ - $($(package)_autoconf) $($(package)_config_opts) RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" -endef - -define $(package)_build_cmds - echo "path is problematic here" &&\ - make -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef - diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk index cdc690e69..462bd2415 100644 --- a/contrib/depends/packages/sodium.mk +++ b/contrib/depends/packages/sodium.mk @@ -1,20 +1,21 @@ package=sodium -$(package)_version=1.0.16 +$(package)_version=1.0.18 $(package)_download_path=https://download.libsodium.org/libsodium/releases/ $(package)_file_name=libsodium-$($(package)_version).tar.gz -$(package)_sha256_hash=eeadc7e1e1bcef09680fb4837d448fbdf57224978f865ac1c16745868fbd0533 +$(package)_sha256_hash=6f504490b342a4f8a4c4a02fc9b866cbef8622d5df4e5452b46be121e46636c1 $(package)_patches=fix-whitespace.patch define $(package)_set_vars $(package)_config_opts=--enable-static --disable-shared $(package)_config_opts+=--prefix=$(host_prefix) $(package)_config_opts_android=RANLIB=$($(package)_ranlib) AR=$($(package)_ar) CC=$($(package)_cc) +$(package)_config_opts_darwin=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" endef define $(package)_config_cmds ./autogen.sh &&\ patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch &&\ - $($(package)_autoconf) $($(package)_config_opts) + $($(package)_autoconf) $($(package)_config_opts) AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk index fddbd0561..826a820c4 100644 --- a/contrib/depends/packages/unwind.mk +++ b/contrib/depends/packages/unwind.mk @@ -3,11 +3,16 @@ $(package)_version=1.2 $(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind $(package)_file_name=lib$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992 +$(package)_patches=fix_obj_order.patch + +define $(package)_preprocess_cmds + patch -p0 < $($(package)_patch_dir)/fix_obj_order.patch +endef define $(package)_config_cmds cp -f $(BASEDIR)/config.guess config/config.guess &&\ cp -f $(BASEDIR)/config.sub config/config.sub &&\ - $($(package)_autoconf) --disable-shared --enable-static + $($(package)_autoconf) --disable-shared --enable-static AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/zeromq.mk b/contrib/depends/packages/zeromq.mk index 46caf14cb..55941e67d 100644 --- a/contrib/depends/packages/zeromq.mk +++ b/contrib/depends/packages/zeromq.mk @@ -8,6 +8,7 @@ $(package)_patches=9114d3957725acd34aa8b8d011585812f3369411.patch 9e6745c12e0b10 define $(package)_set_vars $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve $(package)_config_opts_linux=--with-pic + $(package)_config_opts_freebsd=--with-pic $(package)_cxxflags=-std=c++11 endef @@ -19,7 +20,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/patches/boost/fix_aroptions.patch b/contrib/depends/patches/boost/fix_aroptions.patch new file mode 100644 index 000000000..5b2ec1006 --- /dev/null +++ b/contrib/depends/patches/boost/fix_aroptions.patch @@ -0,0 +1,28 @@ +--- boost_1_64_0/tools/build/src/tools/gcc.jam.O 2017-04-17 03:22:26.000000000 +0100 ++++ boost_1_64_0/tools/build/src/tools/gcc.jam 2019-11-15 15:46:16.957937137 +0000 +@@ -243,6 +243,8 @@ + { + ECHO notice: using gcc archiver :: $(condition) :: $(archiver[1]) ; + } ++ local arflags = [ feature.get-values <arflags> : $(options) ] ; ++ toolset.flags gcc.archive .ARFLAGS $(condition) : $(arflags) ; + + # - Ranlib. + local ranlib = [ common.get-invocation-command gcc +@@ -970,6 +972,7 @@ + # logic in intel-linux, but that is hardly worth the trouble as on Linux, 'ar' + # is always available. + .AR = ar ; ++.ARFLAGS = rc ; + .RANLIB = ranlib ; + + toolset.flags gcc.archive AROPTIONS <archiveflags> ; +@@ -1011,7 +1014,7 @@ + # + actions piecemeal archive + { +- "$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)" ++ "$(.AR)" $(AROPTIONS) $(.ARFLAGS) "$(<)" "$(>)" + "$(.RANLIB)" "$(<)" + } + diff --git a/contrib/depends/patches/native_cctools/skip_otool.patch b/contrib/depends/patches/native_cctools/skip_otool.patch new file mode 100644 index 000000000..30c4ee524 --- /dev/null +++ b/contrib/depends/patches/native_cctools/skip_otool.patch @@ -0,0 +1,12 @@ +--- cctools/Makefile.am.O 2016-06-09 15:06:16.000000000 +0100 ++++ cctools/Makefile.am 2019-11-18 08:59:20.078663220 +0000 +@@ -1,7 +1,7 @@ + if ISDARWIN +-SUBDIRS=libstuff ar as misc otool ld64 $(LD_CLASSIC) ++SUBDIRS=libstuff ar as misc ld64 $(LD_CLASSIC) + else +-SUBDIRS=libstuff ar as misc libobjc2 otool ld64 $(LD_CLASSIC) ++SUBDIRS=libstuff ar as misc ld64 $(LD_CLASSIC) + endif + + ACLOCAL_AMFLAGS = -I m4 diff --git a/contrib/depends/patches/openssl/fix_arflags.patch b/contrib/depends/patches/openssl/fix_arflags.patch new file mode 100644 index 000000000..2d2900d80 --- /dev/null +++ b/contrib/depends/patches/openssl/fix_arflags.patch @@ -0,0 +1,24 @@ +--- Makefile.org.O 2019-02-26 14:20:20.000000000 +0000 ++++ Makefile.org 2019-11-15 13:05:54.370086856 +0000 +@@ -63,8 +63,8 @@ + PEX_LIBS= + EX_LIBS= + EXE_EXT= +-ARFLAGS= +-AR=ar $(ARFLAGS) r ++ARFLAGS= r ++AR=ar $(ARFLAGS) + RANLIB= ranlib + RC= windres + NM= nm +--- Configure.O 2019-02-26 14:20:20.000000000 +0000 ++++ Configure 2019-11-16 07:43:14.933990774 +0000 +@@ -1251,7 +1251,7 @@ + my $shared_extension = $fields[$idx_shared_extension]; + my $ranlib = $ENV{'RANLIB'} || $fields[$idx_ranlib]; + my $ar = $ENV{'AR'} || "ar"; +-my $arflags = $fields[$idx_arflags]; ++my $arflags = $ENV{'ARFLAGS'} || $fields[$idx_arflags]; + my $windres = $ENV{'RC'} || $ENV{'WINDRES'} || "windres"; + my $multilib = $fields[$idx_multilib]; + diff --git a/contrib/depends/patches/sodium/fix-whitespace.patch b/contrib/depends/patches/sodium/fix-whitespace.patch index efbfe4e83..c3d3af0b4 100644 --- a/contrib/depends/patches/sodium/fix-whitespace.patch +++ b/contrib/depends/patches/sodium/fix-whitespace.patch @@ -5,8 +5,8 @@ index b29f769..ca008ae 100755 @@ -591,7 +591,7 @@ MAKEFLAGS= PACKAGE_NAME='libsodium' PACKAGE_TARNAME='libsodium' - PACKAGE_VERSION='1.0.16' --PACKAGE_STRING='libsodium 1.0.16' + PACKAGE_VERSION='1.0.18' +-PACKAGE_STRING='libsodium 1.0.18' +PACKAGE_STRING='libsodium' PACKAGE_BUGREPORT='https://github.com/jedisct1/libsodium/issues' PACKAGE_URL='https://github.com/jedisct1/libsodium' diff --git a/contrib/depends/patches/unwind/fix_obj_order.patch b/contrib/depends/patches/unwind/fix_obj_order.patch new file mode 100644 index 000000000..374a9f04a --- /dev/null +++ b/contrib/depends/patches/unwind/fix_obj_order.patch @@ -0,0 +1,11 @@ +--- config/ltmain.sh.O 2017-01-13 16:00:54.000000000 +0000 ++++ config/ltmain.sh 2019-11-17 06:46:51.994402494 +0000 +@@ -7957,6 +7957,8 @@ + esac + done + fi ++ oldobjs=`for obj in $oldobjs; do echo $obj; done | sort` ++ oldobjs=" `echo $oldobjs`" + eval cmds=\"$old_archive_cmds\" + + func_len " $cmds" diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index 13018ea26..c56f3eb2e 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -1,4 +1,4 @@ -# Set the system name to one of Android, Darwin, Linux, or Windows +# Set the system name to one of Android, Darwin, FreeBSD, Linux, or Windows SET(CMAKE_SYSTEM_NAME @depends@) SET(CMAKE_BUILD_TYPE @release_type@) @@ -23,13 +23,14 @@ SET(Readline_INCLUDE_DIR @prefix@/include) SET(Readline_LIBRARY @prefix@/lib/libreadline.a) SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a) -if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib) +if(NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a) SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a) @@ -40,6 +41,8 @@ SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir") SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library") endif() +endif() + SET(ZMQ_INCLUDE_PATH @prefix@/include) SET(ZMQ_LIB @prefix@/lib/libzmq.a) diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 13747b0c8..1b716fca4 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -605,10 +605,17 @@ eof: std::unique_ptr<boost::thread> m_console_thread; async_console_handler m_console_handler; public: + ~console_handlers_binder() { + stop_handling(); + if (m_console_thread.get() != nullptr) + { + m_console_thread->join(); + } + } + bool start_handling(std::function<std::string(void)> prompt, const std::string& usage_string = "", std::function<void(void)> exit_handler = NULL) { m_console_thread.reset(new boost::thread(boost::bind(&console_handlers_binder::run_handling, this, prompt, usage_string, exit_handler))); - m_console_thread->detach(); return true; } bool start_handling(const std::string &prompt, const std::string& usage_string = "", std::function<void(void)> exit_handler = NULL) diff --git a/contrib/epee/include/md5_l.h b/contrib/epee/include/md5_l.h index a45d91bc8..bc7122650 100644 --- a/contrib/epee/include/md5_l.h +++ b/contrib/epee/include/md5_l.h @@ -85,7 +85,7 @@ namespace md5 MD5Update( &ctx, input, ilen ); MD5Final( output, &ctx); - memset( &ctx, 0, sizeof( MD5_CTX) ); + memwipe( &ctx, sizeof( MD5_CTX )); return true; } diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 5d12f9466..128ff10aa 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -410,7 +410,12 @@ PRAGMA_WARNING_DISABLE_VS(4355) else { _dbg3("[sock " << socket().native_handle() << "] peer closed connection"); - if (m_ready_to_close) + bool do_shutdown = false; + CRITICAL_REGION_BEGIN(m_send_que_lock); + if(!m_send_que.size()) + do_shutdown = true; + CRITICAL_REGION_END(); + if (m_ready_to_close || do_shutdown) shutdown(); } m_ready_to_close = true; @@ -470,6 +475,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { MERROR("SSL handshake failed"); boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); + m_ready_to_close = true; bool do_shutdown = false; CRITICAL_REGION_BEGIN(m_send_que_lock); if(!m_send_que.size()) diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 07ed8157b..0c0653cd6 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -120,6 +120,7 @@ #define BEGIN_JSON_RPC_MAP(uri) else if(query_info.m_URI == uri) \ { \ uint64_t ticks = epee::misc_utils::get_tick_count(); \ + response_info.m_mime_tipe = "application/json"; \ epee::serialization::portable_storage ps; \ if(!ps.load_from_json(query_info.m_body)) \ { \ @@ -148,6 +149,7 @@ #define PREPARE_OBJECTS_FROM_JSON(command_type) \ handled = true; \ + response_info.m_mime_tipe = "application/json"; \ boost::value_initialized<epee::json_rpc::request<command_type::request> > req_; \ epee::json_rpc::request<command_type::request>& req = static_cast<epee::json_rpc::request<command_type::request>&>(req_);\ if(!req.load(ps)) \ diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index e100452ca..59895535f 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -110,7 +110,8 @@ namespace epee constexpr std::size_t size() const noexcept { return len; } constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); } - const T &operator[](size_t idx) const { return ptr[idx]; } + T &operator[](size_t idx) noexcept { return ptr[idx]; } + const T &operator[](size_t idx) const noexcept { return ptr[idx]; } private: T* ptr; diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 06eb9bdaf..b18e04a27 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -34,10 +34,28 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +namespace +{ + template<typename context_t> + void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category) + { + MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") + << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); + } + template<typename context_t> + void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) + { + char buf[32]; + snprintf(buf, sizeof(buf), "command-%u", command); + return on_levin_traffic(context, initiator, sent, error, bytes, buf); + } +} + namespace epee { namespace net_utils { +#if 0 template<class t_arg, class t_result, class t_transport> bool invoke_remote_command2(int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport) { @@ -83,16 +101,18 @@ namespace epee } return true; } +#endif template<class t_arg, class t_result, class t_transport> - bool invoke_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport) + bool invoke_remote_command2(const epee::net_utils::connection_context_base context, int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport) { - + const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); std::string buff_to_send, buff_to_recv; stg.store_to_binary(buff_to_send); + on_levin_traffic(context, true, true, false, buff_to_send.size(), command); int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id); if( res <=0 ) { @@ -102,24 +122,30 @@ namespace epee typename serialization::portable_storage stg_ret; if(!stg_ret.load_from_binary(buff_to_recv)) { + on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); return false; } + on_levin_traffic(context, true, false, false, buff_to_recv.size(), command); return result_struct.load(stg_ret); } template<class t_result, class t_arg, class callback_t, class t_transport> - bool async_invoke_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_transport& transport, const callback_t &cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke_remote_command2(const epee::net_utils::connection_context_base &context, int command, const t_arg& out_struct, t_transport& transport, const callback_t &cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { + const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation std::string buff_to_send; stg.store_to_binary(buff_to_send); + on_levin_traffic(context, true, true, false, buff_to_send.size(), command); int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) { + if (!buff.empty()) + on_levin_traffic(context, true, false, true, buff.size(), command); LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code); cb(code, result_struct, context); return false; @@ -127,16 +153,19 @@ namespace epee serialization::portable_storage stg_ret; if(!stg_ret.load_from_binary(buff)) { + on_levin_traffic(context, true, false, true, buff.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); cb(LEVIN_ERROR_FORMAT, result_struct, context); return false; } if (!result_struct.load(stg_ret)) { + on_levin_traffic(context, true, false, true, buff.size(), command); LOG_ERROR("Failed to load result struct on command " << command); cb(LEVIN_ERROR_FORMAT, result_struct, context); return false; } + on_levin_traffic(context, true, false, false, buff.size(), command); cb(code, result_struct, context); return true; }, inv_timeout); @@ -149,14 +178,15 @@ namespace epee } template<class t_arg, class t_transport> - bool notify_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_transport& transport) + bool notify_remote_command2(const typename t_transport::connection_context &context, int command, const t_arg& out_struct, t_transport& transport) { - + const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); std::string buff_to_send; stg.store_to_binary(buff_to_send); + on_levin_traffic(context, true, true, false, buff_to_send.size(), command); int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id); if(res <=0 ) { @@ -173,6 +203,7 @@ namespace epee serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) { + on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in command " << command); return -1; } @@ -181,9 +212,11 @@ namespace epee if (!static_cast<t_in_type&>(in_struct).load(strg)) { + on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load in_struct in command " << command); return -1; } + on_levin_traffic(context, false, false, false, in_buff.size(), command); int res = cb(command, static_cast<t_in_type&>(in_struct), static_cast<t_out_type&>(out_struct), context); serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); @@ -193,6 +226,7 @@ namespace epee LOG_ERROR("Failed to store_to_binary in command" << command); return -1; } + on_levin_traffic(context, false, true, false, buff_out.size(), command); return res; } @@ -203,15 +237,18 @@ namespace epee serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) { + on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in notify " << command); return -1; } boost::value_initialized<t_in_type> in_struct; if (!static_cast<t_in_type&>(in_struct).load(strg)) { + on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load in_struct in notify " << command); return -1; } + on_levin_traffic(context, false, false, false, in_buff.size(), command); return cb(command, in_struct, context); } @@ -296,6 +333,7 @@ namespace epee #define END_INVOKE_MAP2() \ LOG_ERROR("Unknown command:" << command); \ + on_levin_traffic(context, false, false, true, in_buff.size(), "invalid-command"); \ return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \ } } diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 1be5eb5e1..319c0121b 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -188,8 +188,10 @@ POP_WARNINGS return boost::lexical_cast<std::string>(val); } //---------------------------------------------------------------------------- - inline std::string to_string_hex(uint32_t val) + template<typename T> + inline std::string to_string_hex(const T &val) { + static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); std::stringstream ss; ss << std::hex << val; std::string s; diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index 4fa393283..7e3502bcf 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -167,13 +167,12 @@ If all went well, this produces a number of (uncommitted) `.assert` files in the Checking your work ------------------ -Take a look in the assert files and note the SHA256 checksums listed there. eg for `v0.15.0.0` you should get this checksum: +Take a look in the assert files and note the SHA256 checksums listed there. -``` -2b95118f53d98d542a85f8732b84ba13b3cd20517ccb40332b0edd0ddf4f8c62 monero-x86_64-linux-gnu.tar.gz -``` +You should verify that the checksum that is listed matches each of the binaries you actually built. +This may be done on Linux using the `sha256sum` command or on MacOS using `shasum --algorithm 256` for example. -You should verify that this is really the checksum you get on that file you built. You can also look in the gitian.sigs repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. +You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. Signing assert files @@ -188,8 +187,8 @@ VERSION=v0.15.0.0 gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert +gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert ``` -<!-- TODO: Replace * above with ${VERSION} once gitian builds correct file name --> This will create a `.sig` file for each `.assert` file above (2 files for each platform). diff --git a/contrib/gitian/gitian-android.yml b/contrib/gitian/gitian-android.yml index ce213e056..02614b1a5 100644 --- a/contrib/gitian/gitian-android.yml +++ b/contrib/gitian/gitian-android.yml @@ -39,14 +39,13 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="arm-linux-android aarch64-linux-android" - FAKETIME_HOST_PROGS="clang clang++ ar ranlib nm" + FAKETIME_HOST_PROGS="clang clang++ ar nm" FAKETIME_PROGS="date" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" HOST_LDFLAGS=-static-libstdc++ export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} @@ -89,14 +88,13 @@ script: | } # Faketime for depends so intermediate results are comparable - DUMMYTIME="2000-01-01 12:00:00" export PATH_orig=${PATH} - create_global_faketime_wrappers "$DUMMYTIME" - create_per-host_faketime_wrappers "$DUMMYTIME" + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed - export SOURCE_DATE_EPOCH=`date -d "$DUMMYTIME" +%s` + export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` git config --global core.abbrev 9 cd monero @@ -109,6 +107,7 @@ script: | BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host + export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done @@ -117,12 +116,11 @@ script: | export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - ORIGPATH="$PATH" # Build in a new dir for each host + export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + export PATH=${WRAP_DIR}:${BASEPREFIX}/${i}/native/bin:${PATH_orig} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release make ${MAKEOPTS} diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index b60cc9309..0b36fb4a1 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -8,6 +8,12 @@ import sys gsigs = 'https://github.com/monero-project/gitian.sigs.git' gbrepo = 'https://github.com/devrandom/gitian-builder.git' +platforms = {'l': ['Linux', 'linux', 'tar.bz2'], + 'a': ['Android', 'android', 'tar.bz2'], + 'f': ['FreeBSD', 'freebsd', 'tar.bz2'], + 'w': ['Windows', 'win', 'zip'], + 'm': ['MacOS', 'osx', 'tar.bz2'] } + def setup(): global args, workdir programs = ['apt-cacher-ng', 'ruby', 'git', 'make', 'wget'] @@ -22,11 +28,7 @@ def setup(): if not os.path.isdir('builder'): subprocess.check_call(['git', 'clone', gbrepo, 'builder']) os.chdir('builder') - subprocess.check_call(['git', 'config', 'user.email', 'gitianuser@localhost']) - subprocess.check_call(['git', 'config', 'user.name', 'gitianuser']) - subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576']) - subprocess.check_call(['git', 'fetch', 'origin', '72c51f0bd2adec4eedab4dbd06c9229b9c4eb0e3']) - subprocess.check_call(['git', 'cherry-pick', '72c51f0bd2adec4eedab4dbd06c9229b9c4eb0e3']) + subprocess.check_call(['git', 'checkout', 'c0f77ca018cb5332bfd595e0aff0468f77542c23']) os.makedirs('inputs', exist_ok=True) os.chdir('inputs') if not os.path.isdir('monero'): @@ -52,39 +54,32 @@ def rebuild(): print('\nBuilding Dependencies\n') os.makedirs('../out/' + args.version, exist_ok=True) - if args.linux: - print('\nCompiling ' + args.version + ' Linux') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, 'inputs/monero/contrib/gitian/gitian-linux.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../sigs/', 'inputs/monero/contrib/gitian/gitian-linux.yml']) - subprocess.check_call('mv build/out/monero-*.tar.bz2 ../out/'+args.version, shell=True) - - if args.android: - print('\nCompiling ' + args.version + ' Android') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, 'inputs/monero/contrib/gitian/gitian-android.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-android', '--destination', '../sigs/', 'inputs/monero/contrib/gitian/gitian-android.yml']) - subprocess.check_call('mv build/out/monero-*.tar.bz2 ../out/'+args.version, shell=True) - - if args.windows: - print('\nCompiling ' + args.version + ' Windows') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, 'inputs/monero/contrib/gitian/gitian-win.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win', '--destination', '../sigs/', 'inputs/monero/contrib/gitian/gitian-win.yml']) - subprocess.check_call('mv build/out/monero*.zip ../out/'+args.version, shell=True) - - if args.macos: - print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, 'inputs/monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../sigs/', 'inputs/monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call('mv build/out/monero*.tar.bz2 ../out/'+args.version, shell=True) + + for i in args.os: + if i is 'm' and args.nomac: + continue + + os_name = platforms[i][0] + tag_name = platforms[i][1] + suffix = platforms[i][2] + + print('\nCompiling ' + args.version + ' ' + os_name) + infile = 'inputs/monero/contrib/gitian/gitian-' + tag_name + '.yml' + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, infile]) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-'+tag_name, '--destination', '../sigs/', infile]) + subprocess.check_call('mv build/out/monero-*.' + suffix + ' ../out/'+args.version, shell=True) + print('Moving var/install.log to var/install-' + tag_name + '.log') + subprocess.check_call('mv var/install.log var/install-' + tag_name + '.log', shell=True) + print('Moving var/build.log to var/build-' + tag_name + '.log') + subprocess.check_call('mv var/build.log var/build-' + tag_name + '.log', shell=True) os.chdir(workdir) if args.commit_files: print('\nCommitting '+args.version+' Unsigned Sigs\n') os.chdir('sigs') - subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-android/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-win/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx/'+args.signer]) + for i, v in platforms: + subprocess.check_call(['git', 'add', args.version+'-'+v[1]+'/'+args.signer]) subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) os.chdir(workdir) @@ -109,14 +104,9 @@ def verify(): global args, workdir os.chdir('builder') - print('\nVerifying v'+args.version+' Linux\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../sigs/', '-r', args.version+'-linux', 'inputs/monero/contrib/gitian/gitian-linux.yml']) - print('\nVerifying v'+args.version+' Android\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../sigs/', '-r', args.version+'-android', 'inputs/monero/contrib/gitian/gitian-android.yml']) - print('\nVerifying v'+args.version+' Windows\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../sigs/', '-r', args.version+'-win', 'inputs/monero/contrib/gitian/gitian-win.yml']) - print('\nVerifying v'+args.version+' MacOS\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../sigs/', '-r', args.version+'-osx', 'inputs/monero/contrib/gitian/gitian-osx.yml']) + for i, v in platforms: + print('\nVerifying v'+args.version+' '+v[0]+'\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../sigs/', '-r', args.version+'-'+v[1], 'inputs/monero/contrib/gitian/gitian-'+v[1]+'.yml']) os.chdir(workdir) def main(): @@ -129,7 +119,7 @@ def main(): parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') - parser.add_argument('-o', '--os', dest='os', default='lawm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, a for Android, w for Windows, m for MacOS') + parser.add_argument('-o', '--os', dest='os', default='lafwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, a for Android, f for FreeBSD, w for Windows, m for MacOS') parser.add_argument('-r', '--rebuild', action='store_true', dest='rebuild', help='Redo a Gitian build') parser.add_argument('-R', '--rebuildsign', action='store_true', dest='rebuildsign', help='Redo and sign a Gitian build') parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') @@ -146,11 +136,6 @@ def main(): args = parser.parse_args() workdir = os.getcwd() - args.linux = 'l' in args.os - args.android = 'a' in args.os - args.windows = 'w' in args.os - args.macos = 'm' in args.os - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) if args.buildsign: @@ -177,10 +162,11 @@ def main(): os.environ['LXC_GUEST_IP'] = '10.0.3.5' # Disable MacOS build if no SDK found - if args.macos and not os.path.isfile('builder/inputs/MacOSX10.11.sdk.tar.gz'): - args.macos = False + args.nomac = False + if 'm' in args.os and not os.path.isfile('builder/inputs/MacOSX10.11.sdk.tar.gz'): if args.build: print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.nomac = True script_name = os.path.basename(sys.argv[0]) # Signer and version shouldn't be empty diff --git a/contrib/gitian/gitian-freebsd.yml b/contrib/gitian/gitian-freebsd.yml new file mode 100644 index 000000000..0220b82a5 --- /dev/null +++ b/contrib/gitian/gitian-freebsd.yml @@ -0,0 +1,133 @@ +--- +name: "monero-freebsd-0.15" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "clang-8" +- "gperf" +- "gcc-7" +- "g++-7" +- "gcc" +- "g++" +- "binutils-gold" +- "git" +- "pkg-config" +- "build-essential" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "ca-certificates" +- "python" +- "cmake" +- "ccache" +- "protobuf-compiler" +- "libdbus-1-dev" +- "libharfbuzz-dev" +- "libprotobuf-dev" +- "python3-zmq" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-unknown-freebsd" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="clang-8 clang++-8 date" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ + + export GZIP="-9n" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + export ZERO_AR_DATE=1 + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + WRAPPER=${WRAP_DIR}/${i}-${prog} + echo '#!/usr/bin/env bash' > ${WRAPPER} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAPPER} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAPPER} + echo "export FAKETIME=\"$1\"" >> ${WRAPPER} + echo "$NDKDIR/${ABI}-$prog \$@" >> ${WRAPPER} + chmod +x ${WRAPPER} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed + export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` + + git config --global core.abbrev 9 + cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 + for i in $HOSTS; do + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + + ORIGPATH="$PATH" + # Build in a new dir for each host + export SOURCE_DATE_EPOCH=`date -d ${REFERENCE_DATE}T${REFERENCE_TIME} +%s` + export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} + for i in ${HOSTS}; do + export PATH=${WRAP_DIR}:${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release + make ${MAKEOPTS} + chmod 755 bin/* + cp ../LICENSE bin + chmod 644 bin/LICENSE + DISTNAME=monero-${i}-${version} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 + cd .. + rm -rf build + done + diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index d2124e45c..9c2ebac9b 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -50,14 +50,13 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu" - FAKETIME_HOST_PROGS="gcc g++" - FAKETIME_PROGS="date ar ranlib nm" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="date" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" HOST_LDFLAGS=-static-libstdc++ export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} @@ -105,29 +104,15 @@ script: | # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, # but we can't write there. Instead, create a link here and force it to be included in the - # search paths by wrapping gcc/g++. + # search paths. + # This problem goes away if linux-libc-dev:i386 pkg exists, but it's not in bionic. - mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu - rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm - ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm + mkdir -p $EXTRA_INCLUDES_BASE/i686-linux-gnu + rm -f $WRAP_DIR/extra_includes/i686-linux-gnu/asm + ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-linux-gnu/asm - for prog in gcc g++; do - rm -f ${WRAP_DIR}/${prog} - cat << EOF > ${WRAP_DIR}/${prog} - #!/usr/bin/env bash - REAL="`which -a ${prog}-7 | grep -v ${WRAP_DIR}/${prog} | head -1`" - for var in "\$@" - do - if [ "\$var" = "-m32" ]; then - export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - break - fi - done - \$REAL \$@ - EOF - chmod +x ${WRAP_DIR}/${prog} - done + # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed + export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` git config --global core.abbrev 9 cd monero @@ -140,13 +125,17 @@ script: | BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host + export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 for i in $HOSTS; do EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" if [ -d "$EXTRA_INCLUDES" ]; then - export HOST_ID_SALT="$EXTRA_INCLUDES" + export C_INCLUDE_PATH="$EXTRA_INCLUDES" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES" + else + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH fi make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1 - unset HOST_ID_SALT done # Faketime for binaries @@ -157,9 +146,19 @@ script: | ORIGPATH="$PATH" # Build in a new dir for each host + export SOURCE_DATE_EPOCH=`date -d ${REFERENCE_DATE}T${REFERENCE_TIME} +%s` + export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export C_INCLUDE_PATH="$EXTRA_INCLUDES" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES" + else + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH + fi cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON make ${MAKEOPTS} chmod 755 bin/* diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 1f9928bb5..ecc7d4b59 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -32,10 +32,9 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin11" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage" + FAKETIME_PROGS="ar ranlib date dmg genisoimage python" export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} @@ -92,6 +91,7 @@ script: | tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz # Build dependencies for each host + export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done @@ -104,6 +104,7 @@ script: | ORIGPATH="$PATH" # Build in a new dir for each host + export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 7bfb14a95..bd85695c1 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -22,6 +22,19 @@ packages: - "python" - "rename" - "cmake" +alternatives: +- + package: "i686-w64-mingw32-g++" + path: "/usr/bin/i686-w64-mingw32-g++-posix" +- + package: "i686-w64-mingw32-gcc" + path: "/usr/bin/i686-w64-mingw32-gcc-posix" +- + package: "x86_64-w64-mingw32-g++" + path: "/usr/bin/x86_64-w64-mingw32-g++-posix" +- + package: "x86_64-w64-mingw32-gcc" + path: "/usr/bin/x86_64-w64-mingw32-gcc-posix" remotes: - "url": "https://github.com/monero-project/monero.git" "dir": "monero" @@ -29,13 +42,12 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" - FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" + FAKETIME_HOST_PROGS="windres objcopy" FAKETIME_PROGS="date zip" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} @@ -69,37 +81,15 @@ script: | done } - function create_per-host_linker_wrapper { - # This is only needed for trusty, as the mingw linker leaks a few bytes of - # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 - for i in $HOSTS; do - mkdir -p ${WRAP_DIR}/${i} - for prog in collect2; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} - REAL=$(${i}-gcc -print-prog-name=${prog}) - echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} - echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} - chmod +x ${WRAP_DIR}/${i}/${prog} - done - for prog in gcc g++; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_linker_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} + # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed + export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` + git config --global core.abbrev 9 cd monero # Set the version string that gets added to the tar archive name @@ -111,6 +101,7 @@ script: | BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host + export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 for i in $HOSTS; do EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" if [ -d "$EXTRA_INCLUDES" ]; then @@ -129,6 +120,8 @@ script: | ORIGPATH="$PATH" # Run cmake and make, for each create a new build/ directory, # compile from there, archive, export and delete the archive again + export SOURCE_DATE_EPOCH=`date -d ${REFERENCE_DATE}T${REFERENCE_TIME} +%s` + export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 5c756bcdf..8439bec0b 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -1243,7 +1243,7 @@ bool OS::termSupportsColor(void) { std::string term = getEnvironmentVariable("TERM", ""); return term == "xterm" || term == "xterm-color" || term == "xterm-256color" || term == "screen" || term == "linux" || term == "cygwin" - || term == "screen-256color"; + || term == "screen-256color" || term == "screen.xterm-256color"; } // DateTime diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f332af3d3..d45363e24 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -113,12 +113,10 @@ add_subdirectory(lmdb) add_subdirectory(multisig) add_subdirectory(net) add_subdirectory(hardforks) -if(NOT IOS) - add_subdirectory(blockchain_db) -endif() +add_subdirectory(blockchain_db) add_subdirectory(mnemonics) +add_subdirectory(rpc) if(NOT IOS) - add_subdirectory(rpc) add_subdirectory(serialization) endif() add_subdirectory(wallet) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 63ac38a88..1a6a19da5 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -44,6 +44,71 @@ using epee::string_tools::pod_to_hex; namespace cryptonote { +bool matches_category(relay_method method, relay_category category) noexcept +{ + switch (category) + { + default: + return false; + case relay_category::all: + return true; + case relay_category::relayable: + if (method == relay_method::none) + return false; + return true; + case relay_category::broadcasted: + case relay_category::legacy: + break; + } + // check for "broadcasted" or "legacy" methods: + switch (method) + { + default: + case relay_method::local: + return false; + case relay_method::block: + case relay_method::fluff: + return true; + case relay_method::none: + break; + } + return category == relay_category::legacy; +} + +void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept +{ + kept_by_block = 0; + do_not_relay = 0; + is_local = 0; + + switch (method) + { + case relay_method::none: + do_not_relay = 1; + break; + case relay_method::local: + is_local = 1; + break; + default: + case relay_method::fluff: + break; + case relay_method::block: + kept_by_block = 1; + break; + } +} + +relay_method txpool_tx_meta_t::get_relay_method() const noexcept +{ + if (kept_by_block) + return relay_method::block; + if (do_not_relay) + return relay_method::none; + if (is_local) + return relay_method::local; + return relay_method::fluff; +} + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]." @@ -924,4 +989,23 @@ void BlockchainDB::fixup() batch_stop(); } +bool BlockchainDB::txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category) +{ + try + { + txpool_tx_meta_t meta{}; + if (!get_txpool_tx_meta(tx_hash, meta)) + { + MERROR("Failed to get tx meta from txpool"); + return false; + } + return meta.matches(category); + } + catch (const std::exception &e) + { + MERROR("Failed to get tx meta from txpool: " << e.what()); + } + return false; +} + } // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index d1e4919be..e9fc85803 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -39,6 +39,7 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/difficulty.h" #include "cryptonote_basic/hardfork.h" +#include "cryptonote_protocol/enums.h" /** \file * Cryptonote Blockchain Database Interface @@ -105,6 +106,16 @@ typedef std::pair<crypto::hash, uint64_t> tx_out_index; extern const command_line::arg_descriptor<std::string> arg_db_sync_mode; extern const command_line::arg_descriptor<bool, false> arg_db_salvage; +enum class relay_category : uint8_t +{ + broadcasted = 0,//!< Public txes received via block/fluff + relayable, //!< Every tx not marked `relay_method::none` + legacy, //!< `relay_category::broadcasted` + `relay_method::none` for rpc relay requests or historical reasons + all //!< Everything in the db +}; + +bool matches_category(relay_method method, relay_category category) noexcept; + #pragma pack(push, 1) /** @@ -156,11 +167,22 @@ struct txpool_tx_meta_t uint8_t do_not_relay; uint8_t double_spend_seen: 1; uint8_t pruned: 1; - uint8_t bf_padding: 6; + uint8_t is_local: 1; + uint8_t bf_padding: 5; uint8_t padding[76]; // till 192 bytes + + void set_relay_method(relay_method method) noexcept; + relay_method get_relay_method() const noexcept; + + //! See `relay_category` description + bool matches(const relay_category category) const noexcept + { + return matches_category(get_relay_method(), category); + } }; + #define DBF_SAFE 1 #define DBF_FAST 2 #define DBF_FASTEST 4 @@ -1253,6 +1275,22 @@ public: virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; /** + * @brief fetches a number of pruned transaction blob from the given hash, in canonical blockchain order + * + * The subclass should return the pruned transactions stored from the one with the given + * hash. + * + * If the first transaction does not exist, the subclass should return false. + * If the first transaction exists, but there are fewer transactions starting with it + * than requested, the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transactions were found + */ + virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const = 0; + + /** * @brief fetches the prunable transaction blob with the given hash * * The subclass should return the prunable transaction stored which has the given @@ -1465,12 +1503,12 @@ public: /** * @brief get the number of transactions in the txpool */ - virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const = 0; + virtual uint64_t get_txpool_tx_count(relay_category tx_category = relay_category::broadcasted) const = 0; /** - * @brief check whether a txid is in the txpool + * @brief check whether a txid is in the txpool and meets tx_category requirements */ - virtual bool txpool_has_tx(const crypto::hash &txid) const = 0; + virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const = 0; /** * @brief remove a txpool transaction @@ -1494,10 +1532,11 @@ public: * * @param txid the transaction id of the transation to lookup * @param bd the blob to return + * @param tx_category for filtering out hidden/private txes * - * @return true if the txid was in the txpool, false otherwise + * @return True iff `txid` is in the pool and meets `tx_category` requirements */ - virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const = 0; + virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const = 0; /** * @brief get a txpool transaction's blob @@ -1506,7 +1545,17 @@ public: * * @return the blob for that transaction */ - virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0; + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const = 0; + + /** + * @brief Check if `tx_hash` relay status is in `category`. + * + * @param tx_hash hash of the transaction to lookup + * @param category relay status category to test against + * + * @return True if `tx_hash` latest relay status is in `category`. + */ + bool txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category); /** * @brief prune output data for the given amount @@ -1604,7 +1653,7 @@ public: * * @return false if the function returns false for any transaction, otherwise true */ - virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = true) const = 0; + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0; /** * @brief runs a function over all key images stored diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index f978ef307..5093015f2 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -163,7 +163,15 @@ int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b) { const char *va = (const char*) a->mv_data; const char *vb = (const char*) b->mv_data; - return strcmp(va, vb); + const size_t sz = std::min(a->mv_size, b->mv_size); + int ret = strncmp(va, vb, sz); + if (ret) + return ret; + if (a->mv_size < b->mv_size) + return -1; + if (a->mv_size > b->mv_size) + return 1; + return 0; } } @@ -1771,7 +1779,7 @@ void BlockchainLMDB::update_txpool_tx(const crypto::hash &txid, const txpool_tx_ } } -uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const +uint64_t BlockchainLMDB::get_txpool_tx_count(relay_category category) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1781,7 +1789,7 @@ uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const TXN_PREFIX_RDONLY(); - if (include_unrelayed_txes) + if (category == relay_category::all) { // No filtering, we can get the number of tx the "fast" way MDB_stat db_stats; @@ -1807,7 +1815,7 @@ uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const if (result) throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str())); const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; - if (!meta.do_not_relay) + if (meta.matches(category)) ++num_entries; } } @@ -1816,7 +1824,7 @@ uint64_t BlockchainLMDB::get_txpool_tx_count(bool include_unrelayed_txes) const return num_entries; } -bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const +bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid, relay_category tx_category) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1825,11 +1833,21 @@ bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const RCURSOR(txpool_meta) MDB_val k = {sizeof(txid), (void *)&txid}; - auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET); + MDB_val v; + auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET); if (result != 0 && result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str())); + if (result == MDB_NOTFOUND) + return false; + + bool found = true; + if (tx_category != relay_category::all) + { + const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; + found = meta.matches(tx_category); + } TXN_POSTFIX_RDONLY(); - return result != MDB_NOTFOUND; + return found; } void BlockchainLMDB::remove_txpool_tx(const crypto::hash& txid) @@ -1883,7 +1901,7 @@ bool BlockchainLMDB::get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta return true; } -bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const +bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1893,6 +1911,21 @@ bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::bl MDB_val k = {sizeof(txid), (void *)&txid}; MDB_val v; + + // if filtering, make sure those requirements are met before copying blob + if (tx_category != relay_category::all) + { + auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return false; + if (result != 0) + throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str())); + + const txpool_tx_meta_t& meta = *(const txpool_tx_meta_t*)v.mv_data; + if (!meta.matches(tx_category)) + return false; + } + auto result = mdb_cursor_get(m_cur_txpool_blob, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) return false; @@ -1904,10 +1937,10 @@ bool BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::bl return true; } -cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid) const +cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const { cryptonote::blobdata bd; - if (!get_txpool_tx_blob(txid, bd)) + if (!get_txpool_tx_blob(txid, bd, tx_category)) throw1(DB_ERROR("Tx not found in txpool: ")); return bd; } @@ -2245,7 +2278,7 @@ bool BlockchainLMDB::check_pruning() return prune_worker(prune_mode_check, 0); } -bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const +bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category category) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2269,8 +2302,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str())); const crypto::hash txid = *(const crypto::hash*)k.mv_data; const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; - if (!include_unrelayed_txes && meta.do_not_relay) - // Skipping that tx + if (!meta.matches(category)) continue; const cryptonote::blobdata *passed_bd = NULL; cryptonote::blobdata bd; @@ -3033,6 +3065,48 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd return true; } +bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (!count) + return true; + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_pruned); + + bd.reserve(bd.size() + count); + + MDB_val_set(v, h); + MDB_val result; + int res = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (res == MDB_NOTFOUND) + return false; + if (res) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", res).c_str())); + + const txindex *tip = (const txindex *)v.mv_data; + const uint64_t id = tip->data.tx_id; + MDB_val_set(val_tx_id, id); + MDB_cursor_op op = MDB_SET; + while (count--) + { + res = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, op); + op = MDB_NEXT; + if (res == MDB_NOTFOUND) + return false; + if (res) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx blob", res).c_str())); + bd.emplace_back(reinterpret_cast<char*>(result.mv_data), result.mv_size); + } + + TXN_POSTFIX_RDONLY(); + + return true; +} + bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 61a551476..7c0b4c72c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -254,6 +254,7 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const; virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; @@ -281,12 +282,12 @@ public: virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta); virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta); - virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; - virtual bool txpool_has_tx(const crypto::hash &txid) const; + virtual uint64_t get_txpool_tx_count(relay_category category = relay_category::broadcasted) const; + virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const; virtual void remove_txpool_tx(const crypto::hash& txid); virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; - virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const; - virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata& bd, relay_category tx_category) const; + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const; virtual uint32_t get_blockchain_pruning_seed() const; virtual bool prune_blockchain(uint32_t pruning_seed = 0); virtual bool update_pruning(); @@ -298,7 +299,7 @@ public: virtual uint64_t get_alt_block_count(); virtual void drop_alt_blocks(); - virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const; + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const; virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index ac19fae25..46de38c7e 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -69,6 +69,7 @@ public: virtual cryptonote::blobdata get_block_blob(const crypto::hash& h) const override { return cryptonote::blobdata(); } virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } + virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const { return false; } virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; } virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; } @@ -126,14 +127,14 @@ public: virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) override {} virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {} - virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const override { return 0; } - virtual bool txpool_has_tx(const crypto::hash &txid) const override { return false; } + virtual uint64_t get_txpool_tx_count(relay_category tx_relay = relay_category::broadcasted) const override { return 0; } + virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const override { return false; } virtual void remove_txpool_tx(const crypto::hash& txid) override {} virtual bool get_txpool_tx_meta(const crypto::hash& txid, cryptonote::txpool_tx_meta_t &meta) const override { return false; } - virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const override { return false; } + virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const override { return false; } virtual uint64_t get_database_size() const override { return 0; } - virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const override { return ""; } - virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = false) const override { return false; } + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const override { return ""; } + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; } virtual void add_block( const cryptonote::block& blk , size_t block_weight diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 5d039d7f4..852e9cf4f 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -174,7 +174,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block for(auto& tx_blob: block_entry.txs) { tx_verification_context tvc = AUTO_VAL_INIT(tvc); - core.handle_incoming_tx(tx_blob, tvc, true, true, false); + core.handle_incoming_tx(tx_blob, tvc, relay_method::block, true); if(tvc.m_verifivation_failed) { MERROR("transaction verification failed, tx_id = " diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 2f66d54aa..0d18b8819 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -264,12 +264,12 @@ skip: { throw std::runtime_error("Aborting: tx == null_hash"); } - if (!db->get_tx_blob(tx_id, bd)) + if (!db->get_pruned_tx_blob(tx_id, bd)) { throw std::runtime_error("Aborting: tx not found"); } transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) + if (!parse_and_validate_tx_base_from_blob(bd, tx)) { LOG_PRINT_L0("Bad txn from db"); return 1; diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index ea2237348..29a37e655 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -84,7 +84,7 @@ void set_performance_timer_log_level(el::Level level); #define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> PERF_TIMER_NAME(name)(new tools::LoggingPerformanceTimer(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, el::Level::Info)) #define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000000) #define PERF_TIMER_STOP(name) do { PERF_TIMER_NAME(name).reset(NULL); } while(0) -#define PERF_TIMER_PAUSE(name) PERF_TIMER_NAME(name)->pause() -#define PERF_TIMER_RESUME(name) PERF_TIMER_NAME(name)->resume() +#define PERF_TIMER_PAUSE(name) PERF_TIMER_NAME(name).pause() +#define PERF_TIMER_RESUME(name) PERF_TIMER_NAME(name).resume() } diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 2748c798c..18204eeee 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -37,16 +37,14 @@ static __thread bool is_leaf = false; namespace tools { threadpool::threadpool(unsigned int max_threads) : running(true), active(0) { - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - max = max_threads ? max_threads : tools::get_max_concurrency(); - size_t i = max ? max - 1 : 0; - while(i--) { - threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); - } + create(max_threads); } threadpool::~threadpool() { + destroy(); +} + +void threadpool::destroy() { try { const boost::unique_lock<boost::mutex> lock(mutex); @@ -64,6 +62,23 @@ threadpool::~threadpool() { try { threads[i].join(); } catch (...) { /* ignore */ } } + threads.clear(); +} + +void threadpool::recycle() { + destroy(); + create(max); +} + +void threadpool::create(unsigned int max_threads) { + boost::thread::attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + max = max_threads ? max_threads : tools::get_max_concurrency(); + size_t i = max ? max - 1 : 0; + running = true; + while(i--) { + threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); + } } void threadpool::submit(waiter *obj, std::function<void()> f, bool leaf) { @@ -145,7 +160,7 @@ void threadpool::run(bool flush) { if (!running) break; active++; - e = queue.front(); + e = std::move(queue.front()); queue.pop_front(); lock.unlock(); ++depth; diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 5e490ee7d..a49d0e14f 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -69,12 +69,17 @@ public: // task to finish. void submit(waiter *waiter, std::function<void()> f, bool leaf = false); + // destroy and recreate threads + void recycle(); + unsigned int get_max_concurrency() const; ~threadpool(); private: threadpool(unsigned int max_threads = 0); + void destroy(); + void create(unsigned int max_threads); typedef struct entry { waiter *wo; std::function<void()> f; diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 0bc6ff63c..f620bb53a 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -101,7 +101,7 @@ namespace tools { const char *base = user ? "https://downloads.getmonero.org/" : "https://updates.getmonero.org/"; #ifdef _WIN32 - static const char *extension = strncmp(buildtag.c_str(), "install-", 8) ? ".zip" : ".exe"; + static const char *extension = strncmp(buildtag.c_str(), "source", 6) ? (strncmp(buildtag.c_str(), "install-", 8) ? ".zip" : ".exe") : ".tar.bz2"; #else static const char extension[] = ".tar.bz2"; #endif diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index 1e305b3a6..bb2c5fb40 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -40,6 +40,7 @@ #include <string.h> #include <stdio.h> #include <stdint.h> +#include <memwipe.h> #include "blake256.h" #define U8TO32(p) \ @@ -277,7 +278,7 @@ void hmac_blake256_init(hmac_state *S, const uint8_t *_key, uint64_t keylen) { } blake256_update(&S->outer, pad, 512); - memset(keyhash, 0, 32); + memwipe(keyhash, sizeof(keyhash)); } // keylen = number of bytes @@ -307,7 +308,7 @@ void hmac_blake224_init(hmac_state *S, const uint8_t *_key, uint64_t keylen) { } blake224_update(&S->outer, pad, 512); - memset(keyhash, 0, 32); + memwipe(keyhash, sizeof(keyhash)); } // datalen = number of bits @@ -327,7 +328,7 @@ void hmac_blake256_final(hmac_state *S, uint8_t *digest) { blake256_final(&S->inner, ihash); blake256_update(&S->outer, ihash, 256); blake256_final(&S->outer, digest); - memset(ihash, 0, 32); + memwipe(ihash, sizeof(ihash)); } void hmac_blake224_final(hmac_state *S, uint8_t *digest) { @@ -335,7 +336,7 @@ void hmac_blake224_final(hmac_state *S, uint8_t *digest) { blake224_final(&S->inner, ihash); blake224_update(&S->outer, ihash, 224); blake224_final(&S->outer, digest); - memset(ihash, 0, 32); + memwipe(ihash, sizeof(ihash)); } // keylen = number of bytes; inlen = number of bytes diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index ca127c3ee..134b630f7 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -101,6 +101,9 @@ #define CRYPTONOTE_MEMPOOL_TX_LIVETIME (86400*3) //seconds, three days #define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME 604800 //seconds, one week + +#define CRYPTONOTE_DANDELIONPP_FLUSH_AVERAGE 5 // seconds + // see src/cryptonote_protocol/levin_notify.cpp #define CRYPTONOTE_NOISE_MIN_EPOCH 5 // minutes #define CRYPTONOTE_NOISE_EPOCH_RANGE 30 // seconds diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index d22158dfc..78893fdf2 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -616,7 +616,7 @@ block Blockchain::pop_block_from_blockchain() // that might not be always true. Unlikely though, and always relaying // these again might cause a spike of traffic as many nodes re-relay // all the transactions in a popped block when a reorg happens. - bool r = m_tx_pool.add_tx(tx, tvc, true, true, false, version); + bool r = m_tx_pool.add_tx(tx, tvc, relay_method::block, true, version); if (!r) { LOG_ERROR("Error returning transaction to tx_pool"); @@ -1765,7 +1765,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { cryptonote::tx_memory_pool::tx_details td; cryptonote::blobdata blob; - if (m_tx_pool.have_tx(txid)) + if (m_tx_pool.have_tx(txid, relay_category::legacy)) { if (m_tx_pool.get_transaction_info(txid, td)) { @@ -2498,10 +2498,17 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons block b; CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block"); blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash; - std::vector<crypto::hash> mis; std::vector<cryptonote::blobdata> txs; - get_transactions_blobs(b.tx_hashes, txs, mis, pruned); - CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + if (pruned) + { + CHECK_AND_ASSERT_MES(m_db->get_pruned_tx_blobs_from(b.tx_hashes.front(), b.tx_hashes.size(), txs), false, "Failed to retrieve all transactions needed"); + } + else + { + std::vector<crypto::hash> mis; + get_transactions_blobs(b.tx_hashes, txs, mis, pruned); + CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + } size += blocks.back().first.first.size(); for (const auto &t: txs) size += t.size(); @@ -3641,7 +3648,7 @@ void Blockchain::return_tx_to_pool(std::vector<std::pair<transaction, blobdata>> // all the transactions in a popped block when a reorg happens. const size_t weight = get_transaction_weight(tx.first, tx.second.size()); const crypto::hash tx_hash = get_transaction_hash(tx.first); - if (!m_tx_pool.add_tx(tx.first, tx_hash, tx.second, weight, tvc, true, true, false, version)) + if (!m_tx_pool.add_tx(tx.first, tx_hash, tx.second, weight, tvc, relay_method::block, true, version)) { MERROR("Failed to return taken transaction with hash: " << get_transaction_hash(tx.first) << " to tx_pool"); } @@ -3661,7 +3668,7 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) uint64_t fee; bool relayed, do_not_relay, double_spend_seen, pruned; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned)) + if(m_tx_pool.have_tx(txid, relay_category::all) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -4895,9 +4902,9 @@ void Blockchain::remove_txpool_tx(const crypto::hash &txid) m_db->remove_txpool_tx(txid); } -uint64_t Blockchain::get_txpool_tx_count(bool include_unrelayed_txes) const +uint64_t Blockchain::get_txpool_tx_count(bool include_sensitive) const { - return m_db->get_txpool_tx_count(include_unrelayed_txes); + return m_db->get_txpool_tx_count(include_sensitive ? relay_category::all : relay_category::broadcasted); } bool Blockchain::get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const @@ -4905,19 +4912,24 @@ bool Blockchain::get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t & return m_db->get_txpool_tx_meta(txid, meta); } -bool Blockchain::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const +bool Blockchain::get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const { - return m_db->get_txpool_tx_blob(txid, bd); + return m_db->get_txpool_tx_blob(txid, bd, tx_category); } -cryptonote::blobdata Blockchain::get_txpool_tx_blob(const crypto::hash& txid) const +cryptonote::blobdata Blockchain::get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const { - return m_db->get_txpool_tx_blob(txid); + return m_db->get_txpool_tx_blob(txid, tx_category); } -bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const +bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category tx_category) const { - return m_db->for_all_txpool_txes(f, include_blob, include_unrelayed_txes); + return m_db->for_all_txpool_txes(f, include_blob, tx_category); +} + +bool Blockchain::txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category) +{ + return m_db->txpool_tx_matches_category(tx_hash, category); } void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync) @@ -5074,7 +5086,12 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get return; } const size_t size_needed = 4 + nblocks * (sizeof(crypto::hash) * 2); - if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && checkpoints.size() >= size_needed) + if(checkpoints.size() != size_needed) + { + MERROR("Failed to load hashes - unexpected data size"); + return; + } + else if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP) { p += sizeof(uint32_t); m_blocks_hash_of_hashes.reserve(nblocks); @@ -5098,7 +5115,7 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get CRITICAL_REGION_LOCAL(m_tx_pool); std::vector<transaction> txs; - m_tx_pool.get_transactions(txs); + m_tx_pool.get_transactions(txs, true); size_t tx_weight; uint64_t fee; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 6467031c2..0aecdcb57 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -964,11 +964,12 @@ namespace cryptonote void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); void remove_txpool_tx(const crypto::hash &txid); - uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; + uint64_t get_txpool_tx_count(bool include_sensitive = false) const; bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; - bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const; - cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; - bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = true) const; + bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const; + cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const; + bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category tx_category = relay_category::broadcasted) const; + bool txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category); bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights); @@ -1042,7 +1043,7 @@ namespace cryptonote std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table; - // SHA-3 hashes for each block and for fast pow checking + // Keccak hashes for each block and for fast pow checking std::vector<std::pair<crypto::hash, crypto::hash>> m_blocks_hash_of_hashes; std::vector<std::pair<crypto::hash, uint64_t>> m_blocks_hash_check; std::vector<crypto::hash> m_blocks_txs_check; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 02620996e..5c8f59291 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -29,6 +29,7 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #include <boost/algorithm/string.hpp> +#include <boost/uuid/nil_generator.hpp> #include "string_tools.h" using namespace epee; @@ -83,6 +84,11 @@ namespace cryptonote , "Run in a regression testing mode." , false }; + const command_line::arg_descriptor<bool> arg_keep_fakechain = { + "keep-fakechain" + , "Don't delete any existing database when in fakechain mode." + , false + }; const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty = { "fixed-difficulty" , "Fixed difficulty used for testing." @@ -173,11 +179,6 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; - static const command_line::arg_descriptor<bool> arg_pad_transactions = { - "pad-transactions" - , "Pad relayed transactions to help defend against traffic volume analysis" - , false - }; static const command_line::arg_descriptor<size_t> arg_max_txpool_weight = { "max-txpool-weight" , "Set maximum txpool weight in bytes." @@ -234,8 +235,7 @@ namespace cryptonote m_disable_dns_checkpoints(false), m_update_download(0), m_nettype(UNDEFINED), - m_update_available(false), - m_pad_transactions(false) + m_update_available(false) { m_checkpoints_updating.clear(); set_cryptonote_protocol(pprotocol); @@ -317,6 +317,7 @@ namespace cryptonote command_line::add_arg(desc, arg_testnet_on); command_line::add_arg(desc, arg_stagenet_on); command_line::add_arg(desc, arg_regtest_on); + command_line::add_arg(desc, arg_keep_fakechain); command_line::add_arg(desc, arg_fixed_difficulty); command_line::add_arg(desc, arg_dns_checkpoints); command_line::add_arg(desc, arg_prep_blocks_threads); @@ -332,7 +333,6 @@ namespace cryptonote command_line::add_arg(desc, arg_block_download_max_size); command_line::add_arg(desc, arg_sync_pruned_blocks); command_line::add_arg(desc, arg_max_txpool_weight); - command_line::add_arg(desc, arg_pad_transactions); command_line::add_arg(desc, arg_block_notify); command_line::add_arg(desc, arg_prune_blockchain); command_line::add_arg(desc, arg_reorg_notify); @@ -375,7 +375,6 @@ namespace cryptonote set_enforce_dns_checkpoints(command_line::get_arg(vm, arg_dns_checkpoints)); test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height)); m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); - m_pad_transactions = get_arg(vm, arg_pad_transactions); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) @@ -470,6 +469,7 @@ namespace cryptonote size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain); bool keep_alt_blocks = command_line::get_arg(vm, arg_keep_alt_blocks); + bool keep_fakechain = command_line::get_arg(vm, arg_keep_fakechain); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -511,7 +511,7 @@ namespace cryptonote bool sync_on_blocks = true; uint64_t sync_threshold = 1; - if (m_nettype == FAKECHAIN) + if (m_nettype == FAKECHAIN && !keep_fakechain) { // reset the db by removing the database file before opening it if (!db->remove_data_file(filename)) @@ -753,7 +753,7 @@ namespace cryptonote return false; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash) { tvc = {}; @@ -817,7 +817,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash) { if(!check_tx_syntax(tx)) { @@ -946,23 +946,29 @@ namespace cryptonote return ret; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const epee::span<const tx_blob_entry> tx_blobs, epee::span<tx_verification_context> tvc, relay_method tx_relay, bool relayed) { TRY_ENTRY(); - CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + + if (tx_blobs.size() != tvc.size()) + { + MERROR("tx_blobs and tx_verification_context spans must have equal size"); + return false; + } struct result { bool res; cryptonote::transaction tx; crypto::hash hash; }; std::vector<result> results(tx_blobs.size()); - tvc.resize(tx_blobs.size()); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::vector<tx_blob_entry>::const_iterator it = tx_blobs.begin(); + epee::span<tx_blob_entry>::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { tpool.submit(&waiter, [&, i, it] { try { - results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, keeped_by_block, relayed, do_not_relay); + results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash); } catch (const std::exception &e) { @@ -978,7 +984,7 @@ namespace cryptonote for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { if (!results[i].res) continue; - if(m_mempool.have_tx(results[i].hash)) + if(m_mempool.have_tx(results[i].hash, relay_category::legacy)) { LOG_PRINT_L2("tx " << results[i].hash << "already have transaction in tx_pool"); already_have[i] = true; @@ -993,7 +999,7 @@ namespace cryptonote tpool.submit(&waiter, [&, i, it] { try { - results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, keeped_by_block, relayed, do_not_relay); + results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash); } catch (const std::exception &e) { @@ -1014,7 +1020,7 @@ namespace cryptonote tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res}); } if (!tx_info.empty()) - handle_incoming_tx_accumulated_batch(tx_info, keeped_by_block); + handle_incoming_tx_accumulated_batch(tx_info, tx_relay == relay_method::block); bool ok = true; it = tx_blobs.begin(); @@ -1024,13 +1030,14 @@ namespace cryptonote ok = false; continue; } - if (keeped_by_block) + if (tx_relay == relay_method::block) get_blockchain_storage().on_new_tx_from_block(results[i].tx); if (already_have[i]) continue; const uint64_t weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], keeped_by_block, relayed, do_not_relay); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], tx_relay, relayed); + if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -1044,19 +1051,9 @@ namespace cryptonote CATCH_ENTRY_L0("core::handle_incoming_txs()", false); } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, relay_method tx_relay, bool relayed) { - std::vector<tx_blob_entry> tx_blobs; - tx_blobs.push_back(tx_blob); - std::vector<tx_verification_context> tvcv(1); - bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); - tvc = tvcv[0]; - return r; - } - //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) - { - return handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, keeped_by_block, relayed, do_not_relay); + return handle_incoming_txs({std::addressof(tx_blob), 1}, {std::addressof(tvc), 1}, tx_relay, relayed); } //----------------------------------------------------------------------------------------------- bool core::get_stat_info(core_stat_info& st_inf) const @@ -1246,13 +1243,13 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed) { crypto::hash tx_hash = get_transaction_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); size_t tx_weight = get_transaction_weight(tx, bl.size()); - return add_new_tx(tx, tx_hash, bl, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); + return add_new_tx(tx, tx_hash, bl, tx_weight, tvc, tx_relay, relayed); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1260,9 +1257,9 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed) { - if(m_mempool.have_tx(tx_hash)) + if(m_mempool.have_tx(tx_hash, relay_category::legacy)) { LOG_PRINT_L2("tx " << tx_hash << "already have transaction in tx_pool"); return true; @@ -1275,40 +1272,62 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, tx_relay, relayed, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { - cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - NOTIFY_NEW_TRANSACTIONS::request r; - for (auto it = txs.begin(); it != txs.end(); ++it) + NOTIFY_NEW_TRANSACTIONS::request public_req{}; + NOTIFY_NEW_TRANSACTIONS::request private_req{}; + for (auto& tx : txs) { - r.txs.push_back(it->second); + switch (std::get<2>(tx)) + { + default: + case relay_method::none: + break; + case relay_method::local: + private_req.txs.push_back(std::move(std::get<1>(tx))); + break; + case relay_method::block: + case relay_method::fluff: + public_req.txs.push_back(std::move(std::get<1>(tx))); + break; + } } - get_protocol()->relay_transactions(r, fake_context); - m_mempool.set_relayed(txs); + + /* All txes are sent on randomized timers per connection in + `src/cryptonote_protocol/levin_notify.cpp.` They are either sent with + "white noise" delays or via diffusion (Dandelion++ fluff). So + re-relaying public and private _should_ be acceptable here. */ + const boost::uuids::uuid source = boost::uuids::nil_uuid(); + if (!public_req.txs.empty()) + get_protocol()->relay_transactions(public_req, source, epee::net_utils::zone::public_); + if (!private_req.txs.empty()) + get_protocol()->relay_transactions(private_req, source, epee::net_utils::zone::invalid); } return true; } //----------------------------------------------------------------------------------------------- - void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) + void core::on_transactions_relayed(const epee::span<const cryptonote::blobdata> tx_blobs, const relay_method tx_relay) { - std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; - cryptonote::transaction tx; - crypto::hash tx_hash; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) + std::vector<crypto::hash> tx_hashes{}; + tx_hashes.resize(tx_blobs.size()); + + cryptonote::transaction tx{}; + for (std::size_t i = 0; i < tx_blobs.size(); ++i) { - LOG_ERROR("Failed to parse relayed transaction"); - return; + if (!parse_and_validate_tx_from_blob(tx_blobs[i], tx, tx_hashes[i])) + { + LOG_ERROR("Failed to parse relayed transaction"); + return; + } } - txs.push_back(std::make_pair(tx_hash, std::move(tx_blob))); - m_mempool.set_relayed(txs); + m_mempool.set_relayed(epee::to_span(tx_hashes), tx_relay); } //----------------------------------------------------------------------------------------------- bool core::get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) @@ -1369,7 +1388,7 @@ namespace cryptonote for (const auto &tx_hash: b.tx_hashes) { cryptonote::blobdata txblob; - CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, txblob), "Transaction not found in pool"); + CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, txblob, relay_category::all), "Transaction not found in pool"); bce.txs.push_back({txblob, crypto::null_hash}); } return bce; @@ -1573,14 +1592,14 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transaction(const crypto::hash &id, cryptonote::blobdata& tx) const + bool core::get_pool_transaction(const crypto::hash &id, cryptonote::blobdata& tx, relay_category tx_category) const { - return m_mempool.get_transaction(id, tx); + return m_mempool.get_transaction(id, tx, tx_category); } //----------------------------------------------------------------------------------------------- bool core::pool_has_tx(const crypto::hash &id) const { - return m_mempool.have_tx(id); + return m_mempool.have_tx(id, relay_category::legacy); } //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos, bool include_sensitive_data) const @@ -1639,8 +1658,9 @@ namespace cryptonote << "You can set the level of process detailization through \"set_log <level|categories>\" command," << ENDL << "where <level> is between 0 (no details) and 4 (very verbose), or custom category based levels (eg, *:WARNING)." << ENDL << ENDL - << "Use the \"help\" command to see the list of available commands." << ENDL - << "Use \"help <command>\" to see a command's documentation." << ENDL + << "Use the \"help\" command to see a simplified list of available commands." << ENDL + << "Use the \"help_advanced\" command to see an advanced list of available commands." << ENDL + << "Use \"help_advanced <command>\" to see a command's documentation." << ENDL << "**********************************************************************" << ENDL); m_starter_message_showed = true; } @@ -1883,9 +1903,10 @@ namespace cryptonote } static constexpr double threshold = 1. / (864000 / DIFFICULTY_TARGET_V2); // one false positive every 10 days + static constexpr unsigned int max_blocks_checked = 150; const time_t now = time(NULL); - const std::vector<time_t> timestamps = m_blockchain_storage.get_last_block_timestamps(60); + const std::vector<time_t> timestamps = m_blockchain_storage.get_last_block_timestamps(max_blocks_checked); static const unsigned int seconds[] = { 5400, 3600, 1800, 1200, 600 }; for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) @@ -1897,7 +1918,7 @@ namespace cryptonote MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")"); if (p < threshold) { - MWARNING("There were " << b << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack. Or it could be just sheer bad luck."); + MWARNING("There were " << b << (b == max_blocks_checked ? " or more" : "") << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack. Or it could be just sheer bad luck."); std::shared_ptr<tools::Notify> block_rate_notify = m_block_rate_notify; if (block_rate_notify) diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index f69ac3509..55efb566c 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -35,7 +35,9 @@ #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> +#include "cryptonote_core/i_core_events.h" #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" +#include "cryptonote_protocol/enums.h" #include "storages/portable_storage_template_helper.h" #include "common/download.h" #include "common/command_line.h" @@ -46,6 +48,7 @@ #include "cryptonote_basic/cryptonote_stat_info.h" #include "warnings.h" #include "crypto/hash.h" +#include "span.h" PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -77,7 +80,7 @@ namespace cryptonote * limited to, communication among the Blockchain, the transaction pool, * any miners, and the network. */ - class core: public i_miner_handler + class core final: public i_miner_handler, public i_core_events { public: @@ -115,14 +118,12 @@ namespace cryptonote * * @param tx_blob the tx to handle * @param tvc metadata about the transaction's validity - * @param keeped_by_block if the transaction has been in a block + * @param tx_relay how the transaction was received * @param relayed whether or not the transaction was relayed to us - * @param do_not_relay whether to prevent the transaction from being relayed * * @return true if the transaction was accepted, false otherwise */ - bool handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, relay_method tx_relay, bool relayed); /** * @brief handles a list of incoming transactions @@ -130,15 +131,35 @@ namespace cryptonote * Parses incoming transactions and, if nothing is obviously wrong, * passes them along to the transaction pool * + * @pre `tx_blobs.size() == tvc.size()` + * * @param tx_blobs the txs to handle * @param tvc metadata about the transactions' validity - * @param keeped_by_block if the transactions have been in a block + * @param tx_relay how the transaction was received. * @param relayed whether or not the transactions were relayed to us - * @param do_not_relay whether to prevent the transactions from being relayed * * @return true if the transactions were accepted, false otherwise */ - bool handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(epee::span<const tx_blob_entry> tx_blobs, epee::span<tx_verification_context> tvc, relay_method tx_relay, bool relayed); + + /** + * @brief handles a list of incoming transactions + * + * Parses incoming transactions and, if nothing is obviously wrong, + * passes them along to the transaction pool + * + * @param tx_blobs the txs to handle + * @param tvc metadata about the transactions' validity + * @param tx_relay how the transaction was received. + * @param relayed whether or not the transactions were relayed to us + * + * @return true if the transactions were accepted, false otherwise + */ + bool handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, relay_method tx_relay, bool relayed) + { + tvc.resize(tx_blobs.size()); + return handle_incoming_txs(epee::to_span(tx_blobs), epee::to_mut_span(tvc), tx_relay, relayed); + } /** * @brief handles an incoming block @@ -212,9 +233,10 @@ namespace cryptonote virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce); /** - * @brief called when a transaction is relayed + * @brief called when a transaction is relayed. + * @note Should only be invoked from `levin_notify`. */ - virtual void on_transaction_relayed(const cryptonote::blobdata& tx); + virtual void on_transactions_relayed(epee::span<const cryptonote::blobdata> tx_blobs, relay_method tx_relay) final; /** @@ -440,11 +462,11 @@ namespace cryptonote /** * @copydoc tx_memory_pool::get_transactions - * @param include_unrelayed_txes include unrelayed txes in result + * @param include_sensitive_txes include private transactions * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_txes = false) const; /** * @copydoc tx_memory_pool::get_txpool_backlog @@ -455,34 +477,34 @@ namespace cryptonote /** * @copydoc tx_memory_pool::get_transactions - * @param include_unrelayed_txes include unrelayed txes in result + * @param include_sensitive_txes include private transactions * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const; /** * @copydoc tx_memory_pool::get_transactions - * @param include_unrelayed_txes include unrelayed txes in result + * @param include_sensitive_txes include private transactions * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transaction_stats(struct txpool_stats& stats, bool include_unrelayed_txes = true) const; + bool get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_txes = false) const; /** * @copydoc tx_memory_pool::get_transaction * * @note see tx_memory_pool::get_transaction */ - bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx) const; + bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx, relay_category tx_category) const; /** * @copydoc tx_memory_pool::get_pool_transactions_and_spent_keys_info - * @param include_unrelayed_txes include unrelayed txes in result + * @param include_sensitive_txes include private transactions * * @note see tx_memory_pool::get_pool_transactions_and_spent_keys_info */ - bool get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos, bool include_unrelayed_txes = true) const; + bool get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos, bool include_sensitive_txes = false) const; /** * @copydoc tx_memory_pool::get_pool_for_rpc @@ -770,13 +792,6 @@ namespace cryptonote bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; } /** - * @brief get whether transaction relay should be padded - * - * @return whether transaction relay should be padded - */ - bool pad_transactions() const { return m_pad_transactions; } - - /** * @brief check a set of hashes against the precompiled hash set * * @return number of usable blocks @@ -852,11 +867,11 @@ namespace cryptonote * @param tx_hash the transaction's hash * @param blob the transaction as a blob * @param tx_weight the weight of the transaction + * @param tx_relay how the transaction was received * @param relayed whether or not the transaction was relayed to us - * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed); /** * @brief add a new transaction to the transaction pool @@ -865,15 +880,14 @@ namespace cryptonote * * @param tx the transaction to add * @param tvc return-by-reference metadata about the transaction's verification process - * @param keeped_by_block whether or not the transaction has been in a block + * @param tx_relay how the transaction was received * @param relayed whether or not the transaction was relayed to us - * @param do_not_relay whether to prevent the transaction from being relayed * * @return true if the transaction is already in the transaction pool, * is already in a block on the Blockchain, or is successfully added * to the transaction pool */ - bool add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed); /** * @copydoc Blockchain::add_new_block @@ -929,8 +943,8 @@ namespace cryptonote bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; void set_semantics_failed(const crypto::hash &tx_hash); - bool handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_tx_post(const tx_blob_entry &tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash); + bool handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash); struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); @@ -1081,7 +1095,6 @@ namespace cryptonote bool m_fluffy_blocks_enabled; bool m_offline; - bool m_pad_transactions; std::shared_ptr<tools::Notify> m_block_rate_notify; }; diff --git a/src/cryptonote_core/i_core_events.h b/src/cryptonote_core/i_core_events.h new file mode 100644 index 000000000..f49fbdf07 --- /dev/null +++ b/src/cryptonote_core/i_core_events.h @@ -0,0 +1,44 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_basic/blobdatatype.h" +#include "cryptonote_protocol/enums.h" +#include "span.h" + +namespace cryptonote +{ + struct i_core_events + { + virtual ~i_core_events() noexcept + {} + + virtual void on_transactions_relayed(epee::span<const cryptonote::blobdata> tx_blobs, relay_method tx_relay) = 0; + }; +} diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 392e611e9..1bc475879 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -115,8 +115,10 @@ namespace cryptonote } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) { + const bool kept_by_block = (tx_relay == relay_method::block); + // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -227,7 +229,7 @@ namespace cryptonote crypto::hash max_used_block_id = null_hash; uint64_t max_used_block_height = 0; - cryptonote::txpool_tx_meta_t meta; + cryptonote::txpool_tx_meta_t meta{}; bool ch_inp_res = check_tx_inputs([&tx]()->cryptonote::transaction&{ return tx; }, id, max_used_block_height, max_used_block_id, tvc, kept_by_block); if(!ch_inp_res) { @@ -241,11 +243,10 @@ namespace cryptonote meta.max_used_block_height = 0; meta.last_failed_height = 0; meta.last_failed_id = null_hash; - meta.kept_by_block = kept_by_block; meta.receive_time = receive_time; meta.last_relayed_time = time(NULL); meta.relayed = relayed; - meta.do_not_relay = do_not_relay; + meta.set_relay_method(tx_relay); meta.double_spend_seen = have_tx_keyimges_as_spent(tx); meta.pruned = tx.pruned; meta.bf_padding = 0; @@ -256,15 +257,16 @@ namespace cryptonote m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - m_blockchain.add_txpool_tx(id, blob, meta); - if (!insert_key_images(tx, id, kept_by_block)) + if (!insert_key_images(tx, id, tx_relay)) return false; + + m_blockchain.add_txpool_tx(id, blob, meta); m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); lock.commit(); } catch (const std::exception &e) { - MERROR("transaction already exists at inserting in memory pool: " << e.what()); + MERROR("Error adding transaction to txpool: " << e.what()); return false; } tvc.m_verifivation_impossible = true; @@ -280,7 +282,6 @@ namespace cryptonote { //update transactions container meta.weight = tx_weight; - meta.kept_by_block = kept_by_block; meta.fee = fee; meta.max_used_block_id = max_used_block_id; meta.max_used_block_height = max_used_block_height; @@ -289,7 +290,7 @@ namespace cryptonote meta.receive_time = receive_time; meta.last_relayed_time = time(NULL); meta.relayed = relayed; - meta.do_not_relay = do_not_relay; + meta.set_relay_method(tx_relay); meta.double_spend_seen = false; meta.pruned = tx.pruned; meta.bf_padding = 0; @@ -302,20 +303,21 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); m_blockchain.remove_txpool_tx(id); - m_blockchain.add_txpool_tx(id, blob, meta); - if (!insert_key_images(tx, id, kept_by_block)) + if (!insert_key_images(tx, id, tx_relay)) return false; + + m_blockchain.add_txpool_tx(id, blob, meta); m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); lock.commit(); } catch (const std::exception &e) { - MERROR("internal error: transaction already exists at inserting in memory pool: " << e.what()); + MERROR("internal error: error adding transaction to txpool: " << e.what()); return false; } tvc.m_added_to_pool = true; - if(meta.fee > 0 && !do_not_relay) + if(meta.fee > 0 && tx_relay != relay_method::none) tvc.m_should_be_relayed = true; } @@ -331,7 +333,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) { crypto::hash h = null_hash; size_t blob_size = 0; @@ -339,7 +341,7 @@ namespace cryptonote t_serializable_object_to_blob(tx, bl); if (bl.size() == 0 || !get_transaction_hash(tx, h)) return false; - return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, tx_relay, relayed, version); } //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_txpool_weight() const @@ -375,7 +377,7 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx in txpool"); + MERROR("Failed to find tx_meta in txpool"); return; } // don't prune the kept_by_block ones, they're likely added because we're adding a block with those @@ -384,7 +386,7 @@ namespace cryptonote --it; continue; } - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); cryptonote::transaction_prefix tx; if (!parse_and_validate_tx_prefix_from_blob(txblob, tx)) { @@ -413,17 +415,38 @@ namespace cryptonote MINFO("Pool weight after pruning is larger than limit: " << m_txpool_weight << "/" << bytes); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction_prefix &tx, const crypto::hash &id, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const transaction_prefix &tx, const crypto::hash &id, relay_method tx_relay) { for(const auto& in: tx.vin) { CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false); std::unordered_set<crypto::hash>& kei_image_set = m_spent_key_images[txin.k_image]; - CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block - << ", kei_image_set.size()=" << kei_image_set.size() << ENDL << "txin.k_image=" << txin.k_image << ENDL - << "tx_id=" << id ); - auto ins_res = kei_image_set.insert(id); - CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set"); + + /* If any existing key-image in the set is publicly visible AND this is + not forcibly "kept_by_block", then fail (duplicate key image). If all + existing key images are supposed to be hidden, we silently allow so + that the node doesn't leak knowledge of a local/stem tx. */ + bool visible = false; + if (tx_relay != relay_method::block) + { + for (const crypto::hash& other_id : kei_image_set) + visible |= m_blockchain.txpool_tx_matches_category(other_id, relay_category::legacy); + } + + CHECK_AND_ASSERT_MES(!visible, false, "internal error: tx_relay=" << unsigned(tx_relay) + << ", kei_image_set.size()=" << kei_image_set.size() << ENDL << "txin.k_image=" << txin.k_image << ENDL + << "tx_id=" << id); + + /* If adding a tx (hash) that already exists, fail only if the tx has + been publicly "broadcast" previously. This way, when a private tx is + received for the first time from a remote node, "this" node will + respond as-if it were seen for the first time. LMDB does the + "hard-check" on key-images, so the effect is overwriting the existing + tx_pool metadata and "first seen" time. */ + const bool new_or_previously_private = + kei_image_set.insert(id).second || + !m_blockchain.txpool_tx_matches_category(id, relay_category::legacy); + CHECK_AND_ASSERT_MES(new_or_previously_private, false, "internal error: try to insert duplicate iterator in key_image set"); } ++m_cookie; return true; @@ -475,10 +498,10 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(id, meta)) { - MERROR("Failed to find tx in txpool"); + MERROR("Failed to find tx_meta in txpool"); return false; } - txblob = m_blockchain.get_txpool_tx_blob(id); + txblob = m_blockchain.get_txpool_tx_blob(id, relay_category::all); auto ci = m_parsed_tx_cache.find(id); if (ci != m_parsed_tx_cache.end()) { @@ -533,7 +556,7 @@ namespace cryptonote MERROR("Failed to find tx in txpool"); return false; } - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); auto ci = m_parsed_tx_cache.find(txid); if (ci != m_parsed_tx_cache.end()) { @@ -611,7 +634,7 @@ namespace cryptonote remove.push_back(std::make_pair(txid, meta.weight)); } return true; - }, false); + }, false, relay_category::all); if (!remove.empty()) { @@ -621,7 +644,7 @@ namespace cryptonote const crypto::hash &txid = entry.first; try { - cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid); + cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); cryptonote::transaction_prefix tx; if (!parse_and_validate_tx_prefix_from_blob(bd, tx)) { @@ -649,7 +672,7 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -667,8 +690,7 @@ namespace cryptonote { try { - cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid); - txs.push_back(std::make_pair(txid, bd)); + txs.emplace_back(txid, m_blockchain.get_txpool_tx_blob(txid, relay_category::all), meta.get_relay_method()); } catch (const std::exception &e) { @@ -678,26 +700,27 @@ namespace cryptonote } } return true; - }, false); + }, false, relay_category::relayable); return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) + void tx_memory_pool::set_relayed(const epee::span<const crypto::hash> hashes, const relay_method method) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const time_t now = time(NULL); LockedTXN lock(m_blockchain); - for (auto it = txs.begin(); it != txs.end(); ++it) + for (const auto& hash : hashes) { try { txpool_tx_meta_t meta; - if (m_blockchain.get_txpool_tx_meta(it->first, meta)) + if (m_blockchain.get_txpool_tx_meta(hash, meta)) { meta.relayed = true; meta.last_relayed_time = now; - m_blockchain.update_txpool_tx(it->first, meta); + meta.set_relay_method(method); + m_blockchain.update_txpool_tx(hash, meta); } } catch (const std::exception &e) @@ -709,18 +732,19 @@ namespace cryptonote lock.commit(); } //--------------------------------------------------------------------------------- - size_t tx_memory_pool::get_transactions_count(bool include_unrelayed_txes) const + size_t tx_memory_pool::get_transactions_count(bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - return m_blockchain.get_txpool_tx_count(include_unrelayed_txes); + return m_blockchain.get_txpool_tx_count(include_sensitive); } //--------------------------------------------------------------------------------- - void tx_memory_pool::get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transactions(std::vector<transaction>& txs, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + txs.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) @@ -732,39 +756,42 @@ namespace cryptonote tx.set_hash(txid); txs.push_back(std::move(tx)); return true; - }, true, include_unrelayed_txes); + }, true, category); } //------------------------------------------------------------------ - void tx_memory_pool::get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + txs.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ txs.push_back(txid); return true; - }, false, include_unrelayed_txes); + }, false, category); } //------------------------------------------------------------------ - void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_unrelayed_txes) const + void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); - backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; - }, false, include_unrelayed_txes); + }, false, category); } //------------------------------------------------------------------ - void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_unrelayed_txes) const + void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; std::map<uint64_t, txpool_histo> agebytes; - stats.txs_total = m_blockchain.get_txpool_tx_count(include_unrelayed_txes); + stats.txs_total = m_blockchain.get_txpool_tx_count(include_sensitive); std::vector<uint32_t> weights; weights.reserve(stats.txs_total); m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ @@ -789,7 +816,8 @@ namespace cryptonote if (meta.double_spend_seen) ++stats.num_double_spends; return true; - }, false, include_unrelayed_txes); + }, false, category); + stats.bytes_med = epee::misc_utils::median(weights); if (stats.txs_total > 1) { @@ -847,8 +875,10 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - tx_infos.reserve(m_blockchain.get_txpool_tx_count()); - key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); + const relay_category category = include_sensitive_data ? relay_category::all : relay_category::broadcasted; + const size_t count = m_blockchain.get_txpool_tx_count(include_sensitive_data); + tx_infos.reserve(count); + key_image_infos.reserve(count); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); @@ -879,7 +909,7 @@ namespace cryptonote txi.double_spend_seen = meta.double_spend_seen; tx_infos.push_back(std::move(txi)); return true; - }, true, include_sensitive_data); + }, true, category); txpool_tx_meta_t meta; for (const key_images_container::value_type& kee : m_spent_key_images) { @@ -889,30 +919,13 @@ namespace cryptonote ki.id_hash = epee::string_tools::pod_to_hex(k_image); for (const crypto::hash& tx_id_hash : kei_image_set) { - if (!include_sensitive_data) - { - try - { - if (!m_blockchain.get_txpool_tx_meta(tx_id_hash, meta)) - { - MERROR("Failed to get tx meta from txpool"); - return false; - } - if (!meta.relayed) - // Do not include that transaction if in restricted mode and it's not relayed - continue; - } - catch (const std::exception &e) - { - MERROR("Failed to get tx meta from txpool: " << e.what()); - return false; - } - } - ki.txs_hashes.push_back(epee::string_tools::pod_to_hex(tx_id_hash)); + if (m_blockchain.txpool_tx_matches_category(tx_id_hash, category)) + ki.txs_hashes.push_back(epee::string_tools::pod_to_hex(tx_id_hash)); } + // Only return key images for which we have at least one tx that we can show for them if (!ki.txs_hashes.empty()) - key_image_infos.push_back(ki); + key_image_infos.push_back(std::move(ki)); } return true; } @@ -948,18 +961,19 @@ namespace cryptonote txi.double_spend_seen = meta.double_spend_seen; tx_infos.push_back(txi); return true; - }, true, false); + }, true, relay_category::broadcasted); for (const key_images_container::value_type& kee : m_spent_key_images) { std::vector<crypto::hash> tx_hashes; const std::unordered_set<crypto::hash>& kei_image_set = kee.second; for (const crypto::hash& tx_id_hash : kei_image_set) { - tx_hashes.push_back(tx_id_hash); + if (m_blockchain.txpool_tx_matches_category(tx_id_hash, relay_category::broadcasted)) + tx_hashes.push_back(tx_id_hash); } - const crypto::key_image& k_image = kee.first; - key_image_infos[k_image] = std::move(tx_hashes); + if (!tx_hashes.empty()) + key_image_infos[kee.first] = std::move(tx_hashes); } return true; } @@ -973,19 +987,26 @@ namespace cryptonote for (const auto& image : key_images) { - spent.push_back(m_spent_key_images.find(image) == m_spent_key_images.end() ? false : true); + bool is_spent = false; + const auto found = m_spent_key_images.find(image); + if (found != m_spent_key_images.end()) + { + for (const crypto::hash& tx_hash : found->second) + is_spent |= m_blockchain.txpool_tx_matches_category(tx_hash, relay_category::broadcasted); + } + spent.push_back(is_spent); } return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_transaction(const crypto::hash& id, cryptonote::blobdata& txblob) const + bool tx_memory_pool::get_transaction(const crypto::hash& id, cryptonote::blobdata& txblob, relay_category tx_category) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); try { - return m_blockchain.get_txpool_tx_blob(id, txblob); + return m_blockchain.get_txpool_tx_blob(id, txblob, tx_category); } catch (const std::exception &e) { @@ -1009,11 +1030,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::have_tx(const crypto::hash &id) const + bool tx_memory_pool::have_tx(const crypto::hash &id, relay_category tx_category) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - return m_blockchain.get_db().txpool_has_tx(id); + return m_blockchain.get_db().txpool_has_tx(id, tx_category); } //--------------------------------------------------------------------------------- bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx) const @@ -1032,7 +1053,14 @@ namespace cryptonote bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_spent_key_images.end() != m_spent_key_images.find(key_im); + bool spent = false; + const auto found = m_spent_key_images.find(key_im); + if (found != m_spent_key_images.end()) + { + for (const crypto::hash& tx_hash : found->second) + spent |= m_blockchain.txpool_tx_matches_category(tx_hash, relay_category::broadcasted); + } + return spent; } //--------------------------------------------------------------------------------- void tx_memory_pool::lock() const @@ -1217,13 +1245,14 @@ namespace cryptonote << "weight: " << meta.weight << std::endl << "fee: " << print_money(meta.fee) << std::endl << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl + << "is_local" << (meta.is_local ? 'T' : 'F') << std::endl << "double_spend_seen: " << (meta.double_spend_seen ? 'T' : 'F') << std::endl << "max_used_block_height: " << meta.max_used_block_height << std::endl << "max_used_block_id: " << meta.max_used_block_id << std::endl << "last_failed_height: " << meta.last_failed_height << std::endl << "last_failed_id: " << meta.last_failed_id << std::endl; return true; - }, !short_format); + }, !short_format, relay_category::all); return ss.str(); } @@ -1255,7 +1284,7 @@ namespace cryptonote for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it) { txpool_tx_meta_t meta; - if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta)) + if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta) && !meta.matches(relay_category::legacy)) { MERROR(" failed to find tx meta"); continue; @@ -1304,7 +1333,9 @@ namespace cryptonote } } - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second); + // "local" and "stem" txes are filtered above + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second, relay_category::all); + cryptonote::transaction tx; // Skip transactions that are not ready to be @@ -1379,7 +1410,7 @@ namespace cryptonote remove.insert(txid); } return true; - }, false); + }, false, relay_category::all); size_t n_removed = 0; if (!remove.empty()) @@ -1389,7 +1420,7 @@ namespace cryptonote { try { - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); + cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); cryptonote::transaction tx; if (!parse_and_validate_tx_from_blob(txblob, tx)) // remove pruned ones on startup, they're meant to be temporary { @@ -1450,7 +1481,7 @@ namespace cryptonote remove.push_back(txid); return true; } - if (!insert_key_images(tx, txid, meta.kept_by_block)) + if (!insert_key_images(tx, txid, meta.get_relay_method())) { MFATAL("Failed to insert key images from txpool tx"); return false; @@ -1458,7 +1489,7 @@ namespace cryptonote m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); m_txpool_weight += meta.weight; return true; - }, true); + }, true, relay_category::all); if (!r) return false; } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index dec7e3cd9..f716440ad 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -32,17 +32,20 @@ #include "include_base_utils.h" #include <set> +#include <tuple> #include <unordered_map> #include <unordered_set> #include <queue> #include <boost/serialization/version.hpp> #include <boost/utility.hpp> +#include "span.h" #include "string_tools.h" #include "syncobj.h" #include "math_helper.h" #include "cryptonote_basic/cryptonote_basic_impl.h" #include "cryptonote_basic/verification_context.h" +#include "cryptonote_protocol/enums.h" #include "blockchain_db/blockchain_db.h" #include "crypto/hash.h" #include "rpc/core_rpc_server_commands_defs.h" @@ -105,9 +108,10 @@ namespace cryptonote * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash + * @tx_relay how the transaction was received * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -119,14 +123,13 @@ namespace cryptonote * * @param tx the transaction to be added * @param tvc return-by-reference status about the transaction verification - * @param kept_by_block has this transaction been in a block? + * @tx_relay how the transaction was received * @param relayed was this transaction from the network or a local client? - * @param do_not_relay to avoid relaying the transaction to the network * @param version the version used to create the transaction * * @return true if the transaction passes validations, otherwise false */ - bool add_tx(transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version); /** * @brief takes a transaction with the given hash from the pool @@ -149,10 +152,11 @@ namespace cryptonote * @brief checks if the pool has a transaction with the given hash * * @param id the hash to look for + * @param tx_category a filter for txes * - * @return true if the transaction is in the pool, otherwise false + * @return true if the transaction is in the pool and meets tx_category requirements */ - bool have_tx(const crypto::hash &id) const; + bool have_tx(const crypto::hash &id, relay_category tx_category) const; /** * @brief action to take when notified of a block added to the blockchain @@ -236,37 +240,37 @@ namespace cryptonote * @brief get a list of all transactions in the pool * * @param txs return-by-reference the list of transactions - * @param include_unrelayed_txes include unrelayed txes in the result + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes * */ - void get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; + void get_transactions(std::vector<transaction>& txs, bool include_sensitive = false) const; /** * @brief get a list of all transaction hashes in the pool * * @param txs return-by-reference the list of transactions - * @param include_unrelayed_txes include unrelayed txes in the result + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes * */ - void get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const; + void get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive = false) const; /** * @brief get (weight, fee, receive time) for all transaction in the pool * * @param txs return-by-reference that data - * @param include_unrelayed_txes include unrelayed txes in the result + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes * */ - void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_unrelayed_txes = true) const; + void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive = false) const; /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics - * @param include_unrelayed_txes include unrelayed txes in the result + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes * */ - void get_transaction_stats(struct txpool_stats& stats, bool include_unrelayed_txes = true) const; + void get_transaction_stats(struct txpool_stats& stats, bool include_sensitive = false) const; /** * @brief get information about all transactions and key images in the pool @@ -275,11 +279,12 @@ namespace cryptonote * * @param tx_infos return-by-reference the transactions' information * @param key_image_infos return-by-reference the spent key images' information - * @param include_sensitive_data include unrelayed txes and fields that are sensitive to the node privacy + * @param include_sensitive_data return stempool, anonymity-pool, and unrelayed + * txes and fields that are sensitive to the node privacy * * @return true */ - bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos, bool include_sensitive_data = true) const; + bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos, bool include_sensitive_data = false) const; /** * @brief get information about all transactions and key images in the pool @@ -308,10 +313,11 @@ namespace cryptonote * * @param h the hash of the transaction to get * @param tx return-by-reference the transaction blob requested + * @param tx_relay last relay method us * * @return true if the transaction is found, otherwise false */ - bool get_transaction(const crypto::hash& h, cryptonote::blobdata& txblob) const; + bool get_transaction(const crypto::hash& h, cryptonote::blobdata& txblob, relay_category tx_category) const; /** * @brief get a list of all relayable transactions and their hashes @@ -326,21 +332,22 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; + bool get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * - * @param txs the list of transactions (and their hashes) + * @param hashes list of tx hashes that are about to be relayed + * @param tx_relay update how the tx left this node */ - void set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs); + void set_relayed(epee::span<const crypto::hash> hashes, relay_method tx_relay); /** * @brief get the total number of transactions in the pool * * @return the number of transactions in the pool */ - size_t get_transactions_count(bool include_unrelayed_txes = true) const; + size_t get_transactions_count(bool include_sensitive = false) const; /** * @brief get a string containing human-readable pool information @@ -441,7 +448,7 @@ namespace cryptonote * * @return true on success, false on error */ - bool insert_key_images(const transaction_prefix &tx, const crypto::hash &txid, bool kept_by_block); + bool insert_key_images(const transaction_prefix &tx, const crypto::hash &txid, relay_method tx_relay); /** * @brief remove old transactions from the pool @@ -544,7 +551,7 @@ namespace cryptonote * transaction on the assumption that the original will not be in a * block again. */ - typedef std::unordered_map<crypto::key_image, std::unordered_set<crypto::hash> > key_images_container; + typedef std::unordered_map<crypto::key_image, std::unordered_set<crypto::hash>> key_images_container; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) public: diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 3b456e324..ddbd45a61 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -130,7 +130,7 @@ namespace cryptonote //----------------- i_bc_protocol_layout --------------------------------------- virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context); - virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context); + virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone); //---------------------------------------------------------------------------------- //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context); bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 74ceeb41d..a7bf0c283 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -188,7 +188,7 @@ namespace cryptonote auto connection_time = time(NULL) - cntxt.m_started; ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + cntxt.m_remote_address.str() - << std::setw(20) << std::hex << peer_id + << std::setw(20) << nodetool::peerid_to_string(peer_id) << std::setw(20) << std::hex << support_flags << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) @@ -248,9 +248,7 @@ namespace cryptonote cnx.rpc_port = cntxt.m_rpc_port; cnx.rpc_credits_per_hash = cntxt.m_rpc_credits_per_hash; - std::stringstream peer_id_str; - peer_id_str << std::hex << std::setw(16) << peer_id; - peer_id_str >> cnx.peer_id; + cnx.peer_id = nodetool::peerid_to_string(peer_id); cnx.support_flags = support_flags; @@ -455,7 +453,7 @@ namespace cryptonote for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, true, true, false); + m_core.handle_incoming_tx(*tx_blob_it, tvc, relay_method::block, true); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Block verification failed: transaction verification failed, dropping connection"); @@ -619,7 +617,7 @@ namespace cryptonote { MDEBUG("Incoming tx " << tx_hash << " not in pool, adding"); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, true, true, false) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx(tx_blob, tvc, relay_method::block, true) || tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Block verification failed: transaction verification failed, dropping connection"); drop_connection(context, false, false); @@ -667,13 +665,13 @@ namespace cryptonote drop_connection(context, false, false); m_core.resume_mine(); return 1; - } - + } + size_t tx_idx = 0; for(auto& tx_hash: new_block.tx_hashes) { cryptonote::blobdata txblob; - if(m_core.get_pool_transaction(tx_hash, txblob)) + if(m_core.get_pool_transaction(tx_hash, txblob, relay_category::broadcasted)) { have_tx.push_back({txblob, crypto::null_hash}); } @@ -702,7 +700,7 @@ namespace cryptonote need_tx_indices.push_back(tx_idx); } } - + ++tx_idx; } @@ -909,8 +907,8 @@ namespace cryptonote newtxs.reserve(arg.txs.size()); for (size_t i = 0; i < arg.txs.size(); ++i) { - cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx({arg.txs[i], crypto::null_hash}, tvc, false, true, false); + cryptonote::tx_verification_context tvc{}; + m_core.handle_incoming_tx({arg.txs[i], crypto::null_hash}, tvc, relay_method::fluff, true); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -925,7 +923,7 @@ namespace cryptonote if(arg.txs.size()) { //TODO: add announce usage here - relay_transactions(arg, context); + relay_transactions(arg, context.m_connection_id, context.m_remote_address.get_zone()); } return 1; @@ -1316,7 +1314,7 @@ namespace cryptonote TIME_MEASURE_START(transactions_process_time); num_txs += block_entry.txs.size(); std::vector<tx_verification_context> tvc; - m_core.handle_incoming_txs(block_entry.txs, tvc, true, true, false); + m_core.handle_incoming_txs(block_entry.txs, tvc, relay_method::block, true); if (tvc.size() != block_entry.txs.size()) { LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); @@ -2181,7 +2179,8 @@ skip: MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL << "You are now synchronized with the network. You may now start monero-wallet-cli." << ENDL << ENDL - << "Use the \"help\" command to see the list of available commands." << ENDL + << "Use the \"help\" command to see a simplified list of available commands." << ENDL + << "Use the \"help_advanced\" command to see an advanced list of available commands." << ENDL << "**********************************************************************"); m_sync_timer.pause(); if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) @@ -2344,14 +2343,14 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> - bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) + bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone) { - for(auto& tx_blob : arg.txs) - m_core.on_transaction_relayed(tx_blob); - - // no check for success, so tell core they're relayed unconditionally - m_p2p->send_txs(std::move(arg.txs), exclude_context.m_remote_address.get_zone(), exclude_context.m_connection_id, m_core.pad_transactions()); - return true; + /* Push all outgoing transactions to this function. The behavior needs to + identify how the transaction is going to be relayed, and then update the + local mempool before doing the relay. The code was already updating the + DB twice on received transactions - it is difficult to workaround this + due to the internal design. */ + return m_p2p->send_txs(std::move(arg.txs), zone, source, m_core) != epee::net_utils::zone::invalid; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index a67178c52..978a9ebf3 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -41,7 +41,7 @@ namespace cryptonote struct i_cryptonote_protocol { virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context)=0; - virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context)=0; + virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone)=0; //virtual bool request_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context)=0; }; @@ -54,7 +54,7 @@ namespace cryptonote { return false; } - virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) + virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone) { return false; } diff --git a/src/cryptonote_protocol/enums.h b/src/cryptonote_protocol/enums.h new file mode 100644 index 000000000..2ec622d94 --- /dev/null +++ b/src/cryptonote_protocol/enums.h @@ -0,0 +1,43 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cstdint> + +namespace cryptonote +{ + //! Methods tracking how a tx was received and relayed + enum class relay_method : std::uint8_t + { + none = 0, //!< Received via RPC with `do_not_relay` set + local, //!< Received via RPC; trying to send over i2p/tor, etc. + block, //!< Received in block, takes precedence over others + fluff //!< Received/sent over public networks + }; +} diff --git a/src/cryptonote_protocol/fwd.h b/src/cryptonote_protocol/fwd.h new file mode 100644 index 000000000..616b48be3 --- /dev/null +++ b/src/cryptonote_protocol/fwd.h @@ -0,0 +1,37 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace cryptonote +{ + class core; + struct cryptonote_connection_context; + struct i_core_events; +} + diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 4b41b5bfc..e45c34e02 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -33,6 +33,7 @@ #include <chrono> #include <deque> #include <stdexcept> +#include <utility> #include "common/expect.h" #include "common/varint.h" @@ -43,6 +44,14 @@ #include "net/dandelionpp.h" #include "p2p/net_node.h" +namespace +{ + int get_command_from_message(const cryptonote::blobdata &msg) + { + return msg.size() >= sizeof(epee::levin::bucket_head2) ? SWAP32LE(((epee::levin::bucket_head2*)msg.data())->m_command) : 0; + } +} + namespace cryptonote { namespace levin @@ -57,6 +66,37 @@ namespace levin constexpr const std::chrono::seconds noise_min_delay{CRYPTONOTE_NOISE_MIN_DELAY}; constexpr const std::chrono::seconds noise_delay_range{CRYPTONOTE_NOISE_DELAY_RANGE}; + /* A custom duration is used for the poisson distribution because of the + variance. If 5 seconds is given to `std::poisson_distribution`, 95% of + the values fall between 1-9s in 1s increments (not granular enough). If + 5000 milliseconds is given, 95% of the values fall between 4859ms-5141ms + in 1ms increments (not enough time variance). Providing 20 quarter + seconds yields 95% of the values between 3s-7.25s in 1/4s increments. */ + using fluff_stepsize = std::chrono::duration<std::chrono::milliseconds::rep, std::ratio<1, 4>>; + constexpr const std::chrono::seconds fluff_average_in{CRYPTONOTE_DANDELIONPP_FLUSH_AVERAGE}; + + /*! Bitcoin Core is using 1/2 average seconds for outgoing connections + compared to incoming. The thinking is that the user controls outgoing + connections (Dandelion++ makes similar assumptions in its stem + algorithm). The randomization yields 95% values between 1s-4s in + 1/4s increments. */ + constexpr const fluff_stepsize fluff_average_out{fluff_stepsize{fluff_average_in} / 2}; + + class random_poisson + { + std::poisson_distribution<fluff_stepsize::rep> dist; + public: + explicit random_poisson(fluff_stepsize average) + : dist(average.count() < 0 ? 0 : average.count()) + {} + + fluff_stepsize operator()() + { + crypto::random_device rand{}; + return fluff_stepsize{dist(rand)}; + } + }; + /*! Select a randomized duration from 0 to `range`. The precision will be to the systems `steady_clock`. As an example, supplying 3 seconds to this function will select a duration from [0, 3] seconds, and the increments @@ -129,6 +169,16 @@ namespace levin return fullBlob; } + bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad) + { + const cryptonote::blobdata blob = make_tx_payload(std::move(txs), pad); + p2p.for_connection(destination, [&blob](detail::p2p_context& context) { + on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob)); + return true; + }); + return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(blob), destination); + } + /* The current design uses `asio::strand`s. The documentation isn't as clear as it should be - a `strand` has an internal `mutex` and `bool`. The `mutex` synchronizes thread access and the `bool` is set when a thread is @@ -187,15 +237,18 @@ namespace levin { struct zone { - explicit zone(boost::asio::io_service& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in, bool is_public) + explicit zone(boost::asio::io_service& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in, bool is_public, bool pad_txs) : p2p(std::move(p2p)), noise(std::move(noise_in)), next_epoch(io_service), + flush_txs(io_service), strand(io_service), map(), channels(), + flush_time(std::chrono::steady_clock::time_point::max()), connection_count(0), - is_public(is_public) + is_public(is_public), + pad_txs(pad_txs) { for (std::size_t count = 0; !noise.empty() && count < CRYPTONOTE_NOISE_CHANNELS; ++count) channels.emplace_back(io_service); @@ -204,11 +257,14 @@ namespace levin const std::shared_ptr<connections> p2p; const epee::byte_slice noise; //!< `!empty()` means zone is using noise channels boost::asio::steady_timer next_epoch; + boost::asio::steady_timer flush_txs; boost::asio::io_service::strand strand; net::dandelionpp::connection_map map;//!< Tracks outgoing uuid's for noise channels or Dandelion++ stems std::deque<noise_channel> channels; //!< Never touch after init; only update elements on `noise_channel.strand` + std::chrono::steady_clock::time_point flush_time; //!< Next expected Dandelion++ fluff flush std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time const bool is_public; //!< Zone is public ipv4/ipv6 connections + const bool pad_txs; //!< Pad txs to the next boundary for privacy }; } // detail @@ -245,49 +301,112 @@ namespace levin } }; - //! Sends a message to every active connection - class flood_notify + //! Sends txs on connections with expired timers, and queues callback for next timer expiration (if any). + struct fluff_flush { std::shared_ptr<detail::zone> zone_; - epee::byte_slice message_; // Requires manual copy - boost::uuids::uuid source_; + std::chrono::steady_clock::time_point flush_time_; - public: - explicit flood_notify(std::shared_ptr<detail::zone> zone, epee::byte_slice message, const boost::uuids::uuid& source) - : zone_(std::move(zone)), message_(message.clone()), source_(source) - {} + static void queue(std::shared_ptr<detail::zone> zone, const std::chrono::steady_clock::time_point flush_time) + { + assert(zone != nullptr); + assert(zone->strand.running_in_this_thread()); - flood_notify(flood_notify&&) = default; - flood_notify(const flood_notify& source) - : zone_(source.zone_), message_(source.message_.clone()), source_(source.source_) - {} + detail::zone& this_zone = *zone; + this_zone.flush_time = flush_time; + this_zone.flush_txs.expires_at(flush_time); + this_zone.flush_txs.async_wait(this_zone.strand.wrap(fluff_flush{std::move(zone), flush_time})); + } - void operator()() const + void operator()(const boost::system::error_code error) { if (!zone_ || !zone_->p2p) return; assert(zone_->strand.running_in_this_thread()); - /* The foreach should be quick, but then it iterates and acquires the - same lock for every connection. So do in a strand because two threads - will ping-pong each other with cacheline invalidations. Revisit if - algorithm changes or the locking strategy within the levin config - class changes. */ - - std::vector<boost::uuids::uuid> connections; - connections.reserve(connection_id_reserve_size); - zone_->p2p->foreach_connection([this, &connections] (detail::p2p_context& context) { - /* Only send to outgoing connections when "flooding" over i2p/tor. - Otherwise this makes the tx linkable to a hidden service address, - making things linkable across connections. */ + const bool timer_error = bool(error); + if (timer_error) + { + if (error != boost::system::errc::operation_canceled) + throw boost::system::system_error{error, "fluff_flush timer failed"}; + + // new timer canceled this one set in future + if (zone_->flush_time < flush_time_) + return; + } + + const auto now = std::chrono::steady_clock::now(); + auto next_flush = std::chrono::steady_clock::time_point::max(); + std::vector<std::pair<std::vector<blobdata>, boost::uuids::uuid>> connections{}; + zone_->p2p->foreach_connection([timer_error, now, &next_flush, &connections] (detail::p2p_context& context) + { + if (!context.fluff_txs.empty()) + { + if (context.flush_time <= now || timer_error) // flush on canceled timer + { + context.flush_time = std::chrono::steady_clock::time_point::max(); + connections.emplace_back(std::move(context.fluff_txs), context.m_connection_id); + context.fluff_txs.clear(); + } + else // not flushing yet + next_flush = std::min(next_flush, context.flush_time); + } + else // nothing to flush + context.flush_time = std::chrono::steady_clock::time_point::max(); + return true; + }); + + for (auto& connection : connections) + make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs); + + if (next_flush != std::chrono::steady_clock::time_point::max()) + fluff_flush::queue(std::move(zone_), next_flush); + else + zone_->flush_time = next_flush; // signal that no timer is set + } + }; + + /*! The "fluff" portion of the Dandelion++ algorithm. Every tx is queued + per-connection and flushed with a randomized poisson timer. This + implementation only has one system timer per-zone, and instead tracks + the lowest flush time. */ + struct fluff_notify + { + std::shared_ptr<detail::zone> zone_; + std::vector<blobdata> txs_; + boost::uuids::uuid source_; + + void operator()() + { + if (!zone_ || !zone_->p2p || txs_.empty()) + return; + + assert(zone_->strand.running_in_this_thread()); + + const auto now = std::chrono::steady_clock::now(); + auto next_flush = std::chrono::steady_clock::time_point::max(); + + random_poisson in_duration(fluff_average_in); + random_poisson out_duration(fluff_average_out); + + zone_->p2p->foreach_connection([this, now, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context) + { if (this->source_ != context.m_connection_id && (this->zone_->is_public || !context.m_is_income)) - connections.emplace_back(context.m_connection_id); + { + if (context.fluff_txs.empty()) + context.flush_time = now + (context.m_is_income ? in_duration() : out_duration()); + + next_flush = std::min(next_flush, context.flush_time); + context.fluff_txs.reserve(context.fluff_txs.size() + this->txs_.size()); + for (const blobdata& tx : this->txs_) + context.fluff_txs.push_back(tx); // must copy instead of move (multiple conns) + } return true; }); - for (const boost::uuids::uuid& connection : connections) - zone_->p2p->send(message_.clone(), connection); + if (next_flush < zone_->flush_time) + fluff_flush::queue(std::move(zone_), next_flush); } }; @@ -432,6 +551,10 @@ namespace levin else message = zone_->noise.clone(); + zone_->p2p->for_connection(channel.connection, [&](detail::p2p_context& context) { + on_levin_traffic(context, true, true, false, message.size(), "noise"); + return true; + }); if (zone_->p2p->send(std::move(message), channel.connection)) { if (!channel.queue.empty() && channel.active.empty()) @@ -451,7 +574,7 @@ namespace levin } }; - //! Prepares connections for new channel epoch and sets timer for next epoch + //! Prepares connections for new channel/dandelionpp epoch and sets timer for next epoch struct start_epoch { // Variables allow for Dandelion++ extension @@ -481,8 +604,8 @@ namespace levin }; } // anonymous - notify::notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, bool is_public) - : zone_(std::make_shared<detail::zone>(service, std::move(p2p), std::move(noise), is_public)) + notify::notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, const bool is_public, const bool pad_txs) + : zone_(std::make_shared<detail::zone>(service, std::move(p2p), std::move(noise), is_public, pad_txs)) { if (!zone_->p2p) throw std::logic_error{"cryptonote::levin::notify cannot have nullptr p2p argument"}; @@ -533,9 +656,19 @@ namespace levin channel.next_noise.cancel(); } - bool notify::send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source, const bool pad_txs) + void notify::run_fluff() { if (!zone_) + return; + zone_->flush_txs.cancel(); + } + + bool notify::send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source) + { + if (txs.empty()) + return true; + + if (!zone_) return false; if (!zone_->noise.empty() && !zone_->channels.empty()) @@ -565,12 +698,7 @@ namespace levin } else { - const std::string payload = make_tx_payload(std::move(txs), pad_txs); - epee::byte_slice message = - epee::levin::make_notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(payload)); - - // traditional monero send technique - zone_->strand.dispatch(flood_notify{zone_, std::move(message), source}); + zone_->strand.dispatch(fluff_notify{zone_, std::move(txs), source}); } return true; diff --git a/src/cryptonote_protocol/levin_notify.h b/src/cryptonote_protocol/levin_notify.h index 484243af5..ce652d933 100644 --- a/src/cryptonote_protocol/levin_notify.h +++ b/src/cryptonote_protocol/levin_notify.h @@ -35,6 +35,7 @@ #include "byte_slice.h" #include "cryptonote_basic/blobdatatype.h" +#include "cryptonote_protocol/fwd.h" #include "net/enums.h" #include "span.h" @@ -53,11 +54,6 @@ namespace nodetool namespace cryptonote { - struct cryptonote_connection_context; -} - -namespace cryptonote -{ namespace levin { namespace detail @@ -86,7 +82,7 @@ namespace levin {} //! Construct an instance with available notification `zones`. - explicit notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, bool is_public); + explicit notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, bool is_public, bool pad_txs); notify(const notify&) = delete; notify(notify&&) = default; @@ -108,11 +104,14 @@ namespace levin //! Run the logic for the next stem timeout imemdiately. Only use in testing. void run_stems(); + //! Run the logic for flushing all Dandelion++ fluff queued txs. Only use in testing. + void run_fluff(); + /*! Send txs using `cryptonote_protocol_defs.h` payload format wrapped in a levin header. The message will be sent in a "discreet" manner if enabled - if `!noise.empty()` then the `command`/`payload` will be queued to send at the next available noise interval. Otherwise, a - standard Monero flood notification will be used. + Dandelion++ fluff algorithm will be used. \note Eventually Dandelion++ stem sending will be used here when enabled. @@ -121,12 +120,9 @@ namespace levin \param source The source of the notification. `is_nil()` indicates this node is the source. Dandelion++ will use this to map a source to a particular stem. - \param pad_txs A request to pad txs to help conceal origin via - statistical analysis. Ignored if noise was enabled during - construction. \return True iff the notification is queued for sending. */ - bool send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source, bool pad_txs); + bool send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source); }; } // levin } // net diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index b827221f6..ed6d3af01 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -28,7 +28,6 @@ #include "common/dns_utils.h" #include "common/command_line.h" -#include "version.h" #include "daemon/command_parser_executor.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -154,6 +153,16 @@ bool t_command_parser_executor::print_blockchain_info(const std::vector<std::str } uint64_t start_index = 0; uint64_t end_index = 0; + if (args[0][0] == '-') + { + int64_t nblocks; + if(!epee::string_tools::get_xtype_from_string(nblocks, args[0])) + { + std::cout << "wrong number of blocks" << std::endl; + return false; + } + return m_executor.print_blockchain_info(nblocks, (uint64_t)-nblocks); + } if(!epee::string_tools::get_xtype_from_string(start_index, args[0])) { std::cout << "wrong starter block index parameter" << std::endl; @@ -244,12 +253,15 @@ bool t_command_parser_executor::print_block(const std::vector<std::string>& args bool t_command_parser_executor::print_transaction(const std::vector<std::string>& args) { + bool include_metadata = false; bool include_hex = false; bool include_json = false; // Assumes that optional flags come after mandatory argument <transaction_hash> for (unsigned int i = 1; i < args.size(); ++i) { - if (args[i] == "+hex") + if (args[i] == "+meta") + include_metadata = true; + else if (args[i] == "+hex") include_hex = true; else if (args[i] == "+json") include_json = true; @@ -261,7 +273,7 @@ bool t_command_parser_executor::print_transaction(const std::vector<std::string> } if (args.empty()) { - std::cout << "expected: print_tx <transaction_hash> [+hex] [+json]" << std::endl; + std::cout << "expected: print_tx <transaction_hash> [+meta] [+hex] [+json]" << std::endl; return true; } @@ -269,7 +281,7 @@ bool t_command_parser_executor::print_transaction(const std::vector<std::string> crypto::hash tx_hash; if (parse_hash256(str_hash, tx_hash)) { - m_executor.print_transaction(tx_hash, include_hex, include_json); + m_executor.print_transaction(tx_hash, include_metadata, include_hex, include_json); } return true; @@ -803,8 +815,7 @@ bool t_command_parser_executor::rpc_payments(const std::vector<std::string>& arg bool t_command_parser_executor::version(const std::vector<std::string>& args) { - std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << std::endl; - return true; + return m_executor.version(); } bool t_command_parser_executor::prune_blockchain(const std::vector<std::string>& args) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index ed614a89b..1ca728e39 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -38,6 +38,7 @@ #include "cryptonote_basic/difficulty.h" #include "cryptonote_basic/hardfork.h" #include "rpc/rpc_payment_signature.h" +#include "rpc/rpc_version_str.h" #include <boost/format.hpp> #include <ctime> #include <string> @@ -380,7 +381,7 @@ static void get_metric_prefix(cryptonote::difficulty_type hr, double& hr_d, char prefix = 0; return; } - static const char metric_prefixes[4] = { 'k', 'M', 'G', 'T' }; + static const char metric_prefixes[] = { 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' }; for (size_t i = 0; i < sizeof(metric_prefixes); ++i) { if (hr < 1000000) @@ -664,7 +665,7 @@ bool t_rpc_command_executor::print_connections() { << std::setw(30) << std::left << address << std::setw(8) << (get_address_type_name((epee::net_utils::address_type)info.address_type)) << std::setw(6) << (info.ssl ? "yes" : "no") - << std::setw(20) << epee::string_tools::pad_string(info.peer_id, 16, '0', true) + << std::setw(20) << info.peer_id << std::setw(20) << info.support_flags << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" << std::setw(25) << info.state @@ -743,17 +744,46 @@ bool t_rpc_command_executor::print_net_stats() return true; } -bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index) { +bool t_rpc_command_executor::print_blockchain_info(int64_t start_block_index, uint64_t end_block_index) { cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request req; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response res; epee::json_rpc::error error_resp; + std::string fail_message = "Problem fetching info"; + + // negative: relative to the end + if (start_block_index < 0) + { + cryptonote::COMMAND_RPC_GET_INFO::request ireq; + cryptonote::COMMAND_RPC_GET_INFO::response ires; + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, ires.status); + return true; + } + } + if (start_block_index < 0 && (uint64_t)-start_block_index >= ires.height) + { + tools::fail_msg_writer() << "start offset is larger than blockchain height"; + return true; + } + start_block_index = ires.height + start_block_index; + end_block_index = start_block_index + end_block_index - 1; + } req.start_height = start_block_index; req.end_height = end_block_index; req.fill_pow_hash = false; - std::string fail_message = "Unsuccessful"; - + fail_message = "Failed calling getblockheadersrange"; if (m_is_rpc) { if (!m_rpc_client->json_rpc_request(req, res, "getblockheadersrange", fail_message.c_str())) @@ -939,6 +969,7 @@ bool t_rpc_command_executor::print_block_by_height(uint64_t height, bool include } bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, + bool include_metadata, bool include_hex, bool include_json) { cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; @@ -981,6 +1012,29 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front(); const std::string &pruned_as_hex = (1 == res.txs.size()) ? res.txs.front().pruned_as_hex : ""; const std::string &prunable_as_hex = (1 == res.txs.size()) ? res.txs.front().prunable_as_hex : ""; + // Print metadata if requested + if (include_metadata) + { + if (!res.txs.front().in_pool) + { + tools::msg_writer() << "Block timestamp: " << res.txs.front().block_timestamp << " (" << tools::get_human_readable_timestamp(res.txs.front().block_timestamp) << ")"; + } + cryptonote::blobdata blob; + if (epee::string_tools::parse_hexstr_to_binbuff(pruned_as_hex + prunable_as_hex, blob)) + { + cryptonote::transaction tx; + if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + { + tools::msg_writer() << "Size: " << blob.size(); + tools::msg_writer() << "Weight: " << cryptonote::get_transaction_weight(tx); + } + else + tools::fail_msg_writer() << "Error parsing transaction blob"; + } + else + tools::fail_msg_writer() << "Error parsing transaction from hex"; + } + // Print raw hex if requested if (include_hex) { @@ -2217,7 +2271,7 @@ bool t_rpc_command_executor::sync_info() for (const auto &s: res.spans) if (s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << + tools::success_msg_writer() << address << " " << p.info.peer_id << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << epee::string_tools::pad_string(epee::string_tools::to_string_hex(p.info.pruning_seed), 8) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; @@ -2442,4 +2496,39 @@ bool t_rpc_command_executor::rpc_payments() return true; } +bool t_rpc_command_executor::version() +{ + cryptonote::COMMAND_RPC_GET_INFO::request req; + cryptonote::COMMAND_RPC_GET_INFO::response res; + + const char *fail_message = "Problem fetching info"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/getinfo", fail_message)) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_info(req, res) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.version.empty() || !cryptonote::rpc::is_version_string_valid(res.version)) + { + tools::fail_msg_writer() << "The daemon software version is not available."; + } + else + { + tools::success_msg_writer() << res.version; + } + + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index e8b12cb9b..1754ce32e 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -85,7 +85,7 @@ public: bool print_connections(); - bool print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index); + bool print_blockchain_info(int64_t start_block_index, uint64_t end_block_index); bool set_log_level(int8_t level); @@ -97,7 +97,7 @@ public: bool print_block_by_height(uint64_t height, bool include_hex); - bool print_transaction(crypto::hash transaction_hash, bool include_hex, bool include_json); + bool print_transaction(crypto::hash transaction_hash, bool include_metadata, bool include_hex, bool include_json); bool is_key_image_spent(const crypto::key_image &ki); @@ -163,6 +163,8 @@ public: bool print_net_stats(); + bool version(); + bool set_bootstrap_daemon( const std::string &address, const std::string &username, diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp index 5af4e1a4a..16758215d 100644 --- a/src/daemonizer/posix_fork.cpp +++ b/src/daemonizer/posix_fork.cpp @@ -127,13 +127,18 @@ void fork(const std::string & pidfile) { quit("Unable to open output file: " + output); } +#else + if (open("/dev/null", O_WRONLY) < 0) + { + quit("Unable to open /dev/null"); + } +#endif // Also send standard error to the same log file. if (dup(1) < 0) { quit("Unable to dup output descriptor"); } -#endif } } // namespace posix diff --git a/src/lmdb/table.h b/src/lmdb/table.h index 41a3de296..4ded4ba54 100644 --- a/src/lmdb/table.h +++ b/src/lmdb/table.h @@ -15,8 +15,8 @@ namespace lmdb { char const* const name; const unsigned flags; - MDB_cmp_func const* const key_cmp; - MDB_cmp_func const* const value_cmp; + MDB_cmp_func* const key_cmp; + MDB_cmp_func* const value_cmp; //! \pre `name != nullptr` \return Open table. expect<MDB_dbi> open(MDB_txn& write_txn) const noexcept; diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 58c1717e0..58c5706de 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -161,6 +161,10 @@ namespace nodetool const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", P2P_DEFAULT_LIMIT_RATE_DOWN}; const command_line::arg_descriptor<int64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", -1}; + const command_line::arg_descriptor<bool> arg_pad_transactions = { + "pad-transactions", "Pad relayed transactions to help defend against traffic volume analysis", false + }; + boost::optional<std::vector<proxy>> get_proxies(boost::program_options::variables_map const& vm) { namespace ip = boost::asio::ip; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 8d861ce29..0e9c1c942 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -38,11 +38,13 @@ #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> #include <boost/uuid/uuid.hpp> +#include <chrono> #include <functional> #include <utility> #include <vector> #include "cryptonote_config.h" +#include "cryptonote_protocol/fwd.h" #include "cryptonote_protocol/levin_notify.h" #include "warnings.h" #include "net/abstract_tcp_server2.h" @@ -108,8 +110,16 @@ namespace nodetool template<class base_type> struct p2p_connection_context_t: base_type //t_payload_net_handler::connection_context //public net_utils::connection_context_base { - p2p_connection_context_t(): peer_id(0), support_flags(0), m_in_timedsync(false) {} + p2p_connection_context_t() + : fluff_txs(), + flush_time(std::chrono::steady_clock::time_point::max()), + peer_id(0), + support_flags(0), + m_in_timedsync(false) + {} + std::vector<cryptonote::blobdata> fluff_txs; + std::chrono::steady_clock::time_point flush_time; peerid_type peer_id; uint32_t support_flags; bool m_in_timedsync; @@ -336,7 +346,7 @@ namespace nodetool virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); - virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs); + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core); virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); @@ -539,6 +549,7 @@ namespace nodetool extern const command_line::arg_descriptor<int64_t> arg_limit_rate_up; extern const command_line::arg_descriptor<int64_t> arg_limit_rate_down; extern const command_line::arg_descriptor<int64_t> arg_limit_rate; + extern const command_line::arg_descriptor<bool> arg_pad_transactions; } POP_WARNINGS diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index f8094bfa8..08bc76d26 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -116,6 +116,7 @@ namespace nodetool command_line::add_arg(desc, arg_limit_rate_up); command_line::add_arg(desc, arg_limit_rate_down); command_line::add_arg(desc, arg_limit_rate); + command_line::add_arg(desc, arg_pad_transactions); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -340,6 +341,7 @@ namespace nodetool { bool testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); bool stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + const bool pad_txs = command_line::get_arg(vm, arg_pad_transactions); m_nettype = testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET; network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; @@ -384,7 +386,7 @@ namespace nodetool m_use_ipv6 = command_line::get_arg(vm, arg_p2p_use_ipv6); m_require_ipv4 = !command_line::get_arg(vm, arg_p2p_ignore_ipv4); public_zone.m_notifier = cryptonote::levin::notify{ - public_zone.m_net_server.get_io_service(), public_zone.m_net_server.get_config_shared(), nullptr, true + public_zone.m_net_server.get_io_service(), public_zone.m_net_server.get_config_shared(), nullptr, true, pad_txs }; if (command_line::has_arg(vm, arg_p2p_add_peer)) @@ -495,7 +497,7 @@ namespace nodetool } zone.m_notifier = cryptonote::levin::notify{ - zone.m_net_server.get_io_service(), zone.m_net_server.get_config_shared(), std::move(this_noise), false + zone.m_net_server.get_io_service(), zone.m_net_server.get_config_shared(), std::move(this_noise), false, pad_txs }; } @@ -1022,7 +1024,7 @@ namespace nodetool epee::simple_event ev; std::atomic<bool> hsh_result(false); - bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, zone.m_net_server.get_config_object(), + bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_, COMMAND_HANDSHAKE::ID, arg, zone.m_net_server.get_config_object(), [this, &pi, &ev, &hsh_result, &just_take_peerlist, &context_](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ev.raise();}); @@ -1089,7 +1091,7 @@ namespace nodetool LOG_WARNING_CC(context_, "COMMAND_HANDSHAKE Failed"); m_network_zones.at(context_.m_remote_address.get_zone()).m_net_server.get_config_object().close(context_.m_connection_id); } - else + else if (!just_take_peerlist) { try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) { @@ -1107,7 +1109,7 @@ namespace nodetool m_payload_handler.get_payload_sync_data(arg.payload_data); network_zone& zone = m_network_zones.at(context_.m_remote_address.get_zone()); - bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_TIMED_SYNC::response>(context_.m_connection_id, COMMAND_TIMED_SYNC::ID, arg, zone.m_net_server.get_config_object(), + bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_TIMED_SYNC::response>(context_, COMMAND_TIMED_SYNC::ID, arg, zone.m_net_server.get_config_object(), [this](int code, const typename COMMAND_TIMED_SYNC::response& rsp, p2p_connection_context& context) { context.m_in_timedsync = false; @@ -1368,7 +1370,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to anchor peer: " << peerid_type(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to anchor peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1882,6 +1884,7 @@ namespace nodetool --i; continue; } + local_peerlist[i].last_seen = 0; #ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); @@ -1949,7 +1952,7 @@ namespace nodetool const network_zone& zone = m_network_zones.at(zone_type); if(zone.m_config.m_peer_id != tr.peer_id) { - MWARNING("check_trust failed: peer_id mismatch (passed " << tr.peer_id << ", expected " << zone.m_config.m_peer_id<< ")"); + MWARNING("check_trust failed: peer_id mismatch (passed " << tr.peer_id << ", expected " << peerid_to_string(zone.m_config.m_peer_id) << ")"); return false; } crypto::public_key pk = AUTO_VAL_INIT(pk); @@ -2053,13 +2056,18 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - epee::net_utils::zone node_server<t_payload_net_handler>::send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs) + epee::net_utils::zone node_server<t_payload_net_handler>::send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core) { namespace enet = epee::net_utils; - const auto send = [&txs, &source, pad_txs] (std::pair<const enet::zone, network_zone>& network) + const auto send = [&txs, &source, &core] (std::pair<const enet::zone, network_zone>& network) { - if (network.second.m_notifier.send_txs(std::move(txs), source, (pad_txs || network.first != enet::zone::public_))) + const bool is_public = (network.first == enet::zone::public_); + const cryptonote::relay_method tx_relay = is_public ? + cryptonote::relay_method::fluff : cryptonote::relay_method::local; + + core.on_transactions_relayed(epee::to_span(txs), tx_relay); + if (network.second.m_notifier.send_txs(std::move(txs), source)) return network.first; return enet::zone::invalid; }; @@ -2208,7 +2216,7 @@ namespace nodetool network_zone& zone = m_network_zones.at(address.get_zone()); - bool inv_call_res = epee::net_utils::async_invoke_remote_command2<COMMAND_PING::response>(ping_context.m_connection_id, COMMAND_PING::ID, req, zone.m_net_server.get_config_object(), + bool inv_call_res = epee::net_utils::async_invoke_remote_command2<COMMAND_PING::response>(ping_context, COMMAND_PING::ID, req, zone.m_net_server.get_config_object(), [=](int code, const COMMAND_PING::response& rsp, p2p_connection_context& context) { if(code <= 0) @@ -2220,7 +2228,7 @@ namespace nodetool network_zone& zone = m_network_zones.at(address.get_zone()); if(rsp.status != PING_OK_RESPONSE_STATUS_TEXT || pr != rsp.peer_id) { - LOG_WARNING_CC(ping_context, "back ping invoke wrong response \"" << rsp.status << "\" from" << address.str() << ", hsh_peer_id=" << pr_ << ", rsp.peer_id=" << rsp.peer_id); + LOG_WARNING_CC(ping_context, "back ping invoke wrong response \"" << rsp.status << "\" from" << address.str() << ", hsh_peer_id=" << pr_ << ", rsp.peer_id=" << peerid_to_string(rsp.peer_id)); zone.m_net_server.get_config_object().close(ping_context.m_connection_id); return; } @@ -2252,7 +2260,7 @@ namespace nodetool COMMAND_REQUEST_SUPPORT_FLAGS::request support_flags_request; bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_REQUEST_SUPPORT_FLAGS::response> ( - context.m_connection_id, + context, COMMAND_REQUEST_SUPPORT_FLAGS::ID, support_flags_request, m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object(), @@ -2453,7 +2461,7 @@ namespace nodetool zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { ss << cntxt.m_remote_address.str() - << " \t\tpeer_id " << cntxt.peer_id + << " \t\tpeer_id " << peerid_to_string(cntxt.peer_id) << " \t\tconn_id " << cntxt.m_connection_id << (cntxt.m_is_income ? " INC":" OUT") << std::endl; return true; @@ -2711,12 +2719,12 @@ namespace nodetool if (!check_connection_and_handshake_with_peer(pe.adr, pe.last_seen)) { zone.second.m_peerlist.remove_from_peer_gray(pe); - LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST: address: " << pe.adr.host_str() << " Peer ID: " << peerid_to_string(pe.id)); } else { zone.second.m_peerlist.set_peer_just_seen(pe.id, pe.adr, pe.pruning_seed, pe.rpc_port, pe.rpc_credits_per_hash); - LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_to_string(pe.id)); } } return true; diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 752873666..ed88aa28c 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -34,6 +34,8 @@ #include <utility> #include <vector> #include "cryptonote_basic/blobdatatype.h" +#include "cryptonote_protocol/enums.h" +#include "cryptonote_protocol/fwd.h" #include "net/enums.h" #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" @@ -48,7 +50,7 @@ namespace nodetool struct i_p2p_endpoint { virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; - virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs)=0; + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core)=0; virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; @@ -73,7 +75,7 @@ namespace nodetool { return false; } - virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs) + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core) { return epee::net_utils::zone::invalid; } diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index 44b278589..393bddd05 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -132,7 +132,7 @@ namespace nodetool ss << std::setfill ('0') << std::setw (8) << std::hex << std::noshowbase; for(const peerlist_entry& pe: pl) { - ss << pe.id << "\t" << pe.adr.str() + ss << peerid_to_string(pe.id) << "\t" << pe.adr.str() << " \trpc port " << (pe.rpc_port > 0 ? std::to_string(pe.rpc_port) : "-") << " \trpc credits per hash " << (pe.rpc_credits_per_hash > 0 ? std::to_string(pe.rpc_credits_per_hash) : "-") << " \tpruning seed " << pe.pruning_seed diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index ff6fee95c..80ecc5593 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Adapted from Java code by Sarang Noether +// Paper references are to https://eprint.iacr.org/2017/1066 (revision 1 July 2018) #include <stdlib.h> #include <boost/thread/mutex.hpp> @@ -521,6 +522,7 @@ Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) } PERF_TIMER_STOP_BP(PROVE_v); + // PAPER LINES 41-42 PERF_TIMER_START_BP(PROVE_aLaR); for (size_t j = 0; j < M; ++j) { @@ -566,14 +568,14 @@ try_again: rct::key hash_cache = rct::hash_to_scalar(V); PERF_TIMER_START_BP(PROVE_step1); - // PAPER LINES 38-39 + // PAPER LINES 43-44 rct::key alpha = rct::skGen(); rct::key ve = vector_exponent(aL8, aR8); rct::key A; sc_mul(tmp.bytes, alpha.bytes, INV_EIGHT.bytes); rct::addKeys(A, ve, rct::scalarmultBase(tmp)); - // PAPER LINES 40-42 + // PAPER LINES 45-47 rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); rct::key rho = rct::skGen(); ve = vector_exponent(sL, sR); @@ -581,7 +583,7 @@ try_again: rct::addKeys(S, ve, rct::scalarmultBase(rho)); S = rct::scalarmultKey(S, INV_EIGHT); - // PAPER LINES 43-45 + // PAPER LINES 48-50 rct::key y = hash_cache_mash(hash_cache, A, S); if (y == rct::zero()) { @@ -598,24 +600,20 @@ try_again: } // Polynomial construction by coefficients + // PAPER LINES 70-71 rct::keyV l0 = vector_subtract(aL, z); const rct::keyV &l1 = sL; - // This computes the ugly sum/concatenation from PAPER LINE 65 rct::keyV zero_twos(MN); const rct::keyV zpow = vector_powers(z, M+2); - for (size_t i = 0; i < MN; ++i) + for (size_t j = 0; j < M; ++j) { - zero_twos[i] = rct::zero(); - for (size_t j = 1; j <= M; ++j) - { - if (i >= (j-1)*N && i < j*N) + for (size_t i = 0; i < N; ++i) { - CHECK_AND_ASSERT_THROW_MES(1+j < zpow.size(), "invalid zpow index"); - CHECK_AND_ASSERT_THROW_MES(i-(j-1)*N < twoN.size(), "invalid twoN index"); - sc_muladd(zero_twos[i].bytes, zpow[1+j].bytes, twoN[i-(j-1)*N].bytes, zero_twos[i].bytes); + CHECK_AND_ASSERT_THROW_MES(j+2 < zpow.size(), "invalid zpow index"); + CHECK_AND_ASSERT_THROW_MES(i < twoN.size(), "invalid twoN index"); + sc_mul(zero_twos[j*N+i].bytes,zpow[j+2].bytes,twoN[i].bytes); } - } } rct::keyV r0 = vector_add(aR, z); @@ -624,7 +622,7 @@ try_again: r0 = vector_add(r0, zero_twos); rct::keyV r1 = hadamard(yMN, sR); - // Polynomial construction before PAPER LINE 46 + // Polynomial construction before PAPER LINE 51 rct::key t1_1 = inner_product(l0, r1); rct::key t1_2 = inner_product(l1, r0); rct::key t1; @@ -634,7 +632,7 @@ try_again: PERF_TIMER_STOP_BP(PROVE_step1); PERF_TIMER_START_BP(PROVE_step2); - // PAPER LINES 47-48 + // PAPER LINES 52-53 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); rct::key T1, T2; @@ -648,7 +646,7 @@ try_again: ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); ge_p3_tobytes(T2.bytes, &p3); - // PAPER LINES 49-51 + // PAPER LINES 54-56 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); if (x == rct::zero()) { @@ -657,7 +655,7 @@ try_again: goto try_again; } - // PAPER LINES 52-53 + // PAPER LINES 61-63 rct::key taux; sc_mul(taux.bytes, tau1.bytes, x.bytes); rct::key xsq; @@ -671,7 +669,7 @@ try_again: rct::key mu; sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); - // PAPER LINES 54-57 + // PAPER LINES 58-60 rct::keyV l = l0; l = vector_add(l, vector_scalar(l1, x)); rct::keyV r = r0; @@ -690,7 +688,7 @@ try_again: CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); #endif - // PAPER LINES 32-33 + // PAPER LINE 6 rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); if (x_ip == rct::zero()) { @@ -725,20 +723,19 @@ try_again: PERF_TIMER_STOP_BP(PROVE_step3); PERF_TIMER_START_BP(PROVE_step4); - // PAPER LINE 13 const rct::keyV *scale = &yinvpow; while (nprime > 1) { - // PAPER LINE 15 + // PAPER LINE 20 nprime /= 2; - // PAPER LINES 16-17 + // PAPER LINES 21-22 PERF_TIMER_START_BP(PROVE_inner_product); rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); PERF_TIMER_STOP_BP(PROVE_inner_product); - // PAPER LINES 18-19 + // PAPER LINES 23-24 PERF_TIMER_START_BP(PROVE_LR); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); L[round] = cross_vector_exponent8(nprime, Gprime, nprime, Hprime, 0, aprime, 0, bprime, nprime, scale, &ge_p3_H, &tmp); @@ -746,7 +743,7 @@ try_again: R[round] = cross_vector_exponent8(nprime, Gprime, 0, Hprime, nprime, aprime, nprime, bprime, 0, scale, &ge_p3_H, &tmp); PERF_TIMER_STOP_BP(PROVE_LR); - // PAPER LINES 21-22 + // PAPER LINES 25-27 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); if (w[round] == rct::zero()) { @@ -755,7 +752,7 @@ try_again: goto try_again; } - // PAPER LINES 24-25 + // PAPER LINES 29-30 const rct::key winv = invert(w[round]); if (nprime > 1) { @@ -765,7 +762,7 @@ try_again: PERF_TIMER_STOP_BP(PROVE_hadamard2); } - // PAPER LINES 28-29 + // PAPER LINES 33-34 PERF_TIMER_START_BP(PROVE_prime); aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); @@ -776,7 +773,6 @@ try_again: } PERF_TIMER_STOP_BP(PROVE_step4); - // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) return Bulletproof(std::move(V), A, S, T1, T2, taux, mu, std::move(L), std::move(R), aprime[0], bprime[0], t); } @@ -810,7 +806,10 @@ struct proof_data_t size_t logM, inv_offset; }; -/* Given a range proof, determine if it is valid */ +/* Given a range proof, determine if it is valid + * This uses the method in PAPER LINES 95-105, + * weighted across multiple proofs in a batch + */ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) { init_exponents(); @@ -871,7 +870,6 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 // The inner product challenges are computed per round pd.w.resize(rounds); for (size_t i = 0; i < rounds; ++i) @@ -929,7 +927,6 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) rct::key proof8_A = rct::scalarmult8(proof.A); PERF_TIMER_START_BP(VERIFY_line_61); - // PAPER LINE 61 sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes); const rct::keyV zpow = vector_powers(pd.z, M+3); @@ -962,7 +959,6 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) PERF_TIMER_STOP_BP(VERIFY_line_61rl_new); PERF_TIMER_START_BP(VERIFY_line_62); - // PAPER LINE 62 multiexp_data.emplace_back(weight_z, proof8_A); sc_mul(tmp.bytes, pd.x.bytes, weight_z.bytes); multiexp_data.emplace_back(tmp, proof8_S); @@ -973,7 +969,6 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); PERF_TIMER_START_BP(VERIFY_line_24_25); - // Basically PAPER LINES 24-25 // Compute the curvepoints from G[i] and H[i] rct::key yinvpow = rct::identity(); rct::key ypow = rct::identity(); @@ -1010,7 +1005,6 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) sc_mul(g_scalar.bytes, g_scalar.bytes, w_cache[i].bytes); sc_mul(h_scalar.bytes, h_scalar.bytes, w_cache[(~i) & (MN-1)].bytes); - // Adjust the scalars using the exponents from PAPER LINE 62 sc_add(g_scalar.bytes, g_scalar.bytes, pd.z.bytes); CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); @@ -1043,7 +1037,6 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) PERF_TIMER_STOP_BP(VERIFY_line_24_25); - // PAPER LINE 26 PERF_TIMER_START_BP(VERIFY_line_26_new); sc_muladd(z1.bytes, proof.mu.bytes, weight_z.bytes, z1.bytes); for (size_t i = 0; i < rounds; ++i) diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 2c4e5fc3b..1763542db 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -31,6 +31,7 @@ #include "misc_log_ex.h" #include "cryptonote_config.h" #include "rctTypes.h" +#include "int-util.h" using namespace crypto; using namespace std; @@ -118,40 +119,22 @@ namespace rct { //uint long long to 32 byte key void d2h(key & amounth, const xmr_amount in) { sc_0(amounth.bytes); - xmr_amount val = in; - int i = 0; - while (val != 0) { - amounth[i] = (unsigned char)(val & 0xFF); - i++; - val /= (xmr_amount)256; - } + memcpy_swap64le(amounth.bytes, &in, 1); } //uint long long to 32 byte key key d2h(const xmr_amount in) { key amounth; - sc_0(amounth.bytes); - xmr_amount val = in; - int i = 0; - while (val != 0) { - amounth[i] = (unsigned char)(val & 0xFF); - i++; - val /= (xmr_amount)256; - } + d2h(amounth, in); return amounth; } //uint long long to int[64] void d2b(bits amountb, xmr_amount val) { int i = 0; - while (val != 0) { - amountb[i] = val & 1; - i++; - val >>= 1; - } while (i < 64) { - amountb[i] = 0; - i++; + amountb[i++] = val & 1; + val >>= 1; } } @@ -172,16 +155,11 @@ namespace rct { int val = 0, i = 0, j = 0; for (j = 0; j < 8; j++) { val = (unsigned char)test.bytes[j]; - i = 8 * j; - while (val != 0) { - amountb2[i] = val & 1; - i++; + i = 0; + while (i < 8) { + amountb2[j*8+i++] = val & 1; val >>= 1; } - while (i < 8 * (j + 1)) { - amountb2[i] = 0; - i++; - } } } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index ebb1e767f..65d88b57e 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -37,6 +37,7 @@ set(rpc_sources bootstrap_daemon.cpp core_rpc_server.cpp rpc_payment.cpp + rpc_version_str.cpp instanciations) set(daemon_messages_sources @@ -54,6 +55,7 @@ set(rpc_base_headers rpc_handler.h) set(rpc_headers + rpc_version_str.h rpc_handler.h) set(daemon_rpc_server_headers) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 9117b5b3a..e92ae7c08 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -29,6 +29,7 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #include <boost/preprocessor/stringize.hpp> +#include <boost/uuid/nil_generator.hpp> #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -86,10 +87,14 @@ namespace RPCTracker(const char *rpc, tools::LoggingPerformanceTimer &timer): rpc(rpc), timer(timer) { } ~RPCTracker() { - boost::unique_lock<boost::mutex> lock(mutex); - auto &e = tracker[rpc]; - ++e.count; - e.time += timer.value(); + try + { + boost::unique_lock<boost::mutex> lock(mutex); + auto &e = tracker[rpc]; + ++e.count; + e.time += timer.value(); + } + catch (...) { /* ignore */ } } void pay(uint64_t amount) { boost::unique_lock<boost::mutex> lock(mutex); @@ -953,18 +958,21 @@ namespace cryptonote { e.double_spend_seen = it->second.double_spend_seen; e.relayed = it->second.relayed; + e.received_timestamp = it->second.receive_time; } else { MERROR("Failed to determine pool info for " << tx_hash); e.double_spend_seen = false; e.relayed = false; + e.received_timestamp = 0; } } else { e.block_height = m_core.get_blockchain_storage().get_db().get_tx_block_height(tx_hash); e.block_timestamp = m_core.get_blockchain_storage().get_db().get_block_timestamp(e.block_height); + e.received_timestamp = 0; e.double_spend_seen = false; e.relayed = false; } @@ -1095,9 +1103,8 @@ namespace cryptonote } res.sanity_check_failed = false; - cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) + tx_verification_context tvc{}; + if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, (req.do_not_relay ? relay_method::none : relay_method::local), false) || tvc.m_verifivation_failed) { res.status = "Failed"; std::string reason = ""; @@ -1142,7 +1149,7 @@ namespace cryptonote NOTIFY_NEW_TRANSACTIONS::request r; r.txs.push_back(tx_blob); - m_core.get_protocol()->relay_transactions(r, fake_context); + m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes res.status = CORE_RPC_STATUS_OK; return true; @@ -1958,7 +1965,11 @@ namespace cryptonote m_was_bootstrap_ever_used = true; } - r = r && res.status == CORE_RPC_STATUS_OK; + if (r && res.status != CORE_RPC_STATUS_PAYMENT_REQUIRED && res.status != CORE_RPC_STATUS_OK) + { + MINFO("Failing RPC " << command_name << " due to peer return status " << res.status); + r = false; + } res.untrusted = true; return true; } @@ -2222,8 +2233,7 @@ namespace cryptonote //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { - on_get_info(req, res, ctx); - if (res.status != CORE_RPC_STATUS_OK) + if (!on_get_info(req, res, ctx) || res.status != CORE_RPC_STATUS_OK) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = res.status; @@ -2370,7 +2380,7 @@ namespace cryptonote if (req.txids.empty()) { std::vector<transaction> pool_txs; - bool r = m_core.get_pool_transactions(pool_txs); + bool r = m_core.get_pool_transactions(pool_txs, true); if (!r) { res.status = "Failed to get txpool contents"; @@ -2617,6 +2627,7 @@ namespace cryptonote { RPC_TRACKER(update); + res.update = false; if (m_core.offline()) { res.status = "Daemon is running offline"; @@ -2747,13 +2758,11 @@ namespace cryptonote crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); cryptonote::blobdata txblob; - bool r = m_core.get_pool_transaction(txid, txblob); - if (r) + if (!m_core.get_pool_transaction(txid, txblob, relay_category::legacy)) { - cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); NOTIFY_NEW_TRANSACTIONS::request r; r.txs.push_back(txblob); - m_core.get_protocol()->relay_transactions(r, fake_context); + m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes } else @@ -2940,7 +2949,7 @@ namespace cryptonote RPC_TRACKER(rpc_access_info); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_INFO>(invoke_http_mode::JON, "rpc_access_info", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_INFO>(invoke_http_mode::JON_RPC, "rpc_access_info", req, res, r)) return r; // if RPC payment is not enabled @@ -3012,7 +3021,7 @@ namespace cryptonote { RPC_TRACKER(rpc_access_submit_nonce); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_SUBMIT_NONCE>(invoke_http_mode::JON, "rpc_access_submit_nonce", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_SUBMIT_NONCE>(invoke_http_mode::JON_RPC, "rpc_access_submit_nonce", req, res, r)) return r; // if RPC payment is not enabled @@ -3071,7 +3080,7 @@ namespace cryptonote RPC_TRACKER(rpc_access_pay); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_PAY>(invoke_http_mode::JON, "rpc_access_pay", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_PAY>(invoke_http_mode::JON_RPC, "rpc_access_pay", req, res, r)) return r; // if RPC payment is not enabled @@ -3130,7 +3139,7 @@ namespace cryptonote RPC_TRACKER(rpc_access_data); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_DATA>(invoke_http_mode::JON, "rpc_access_data", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_DATA>(invoke_http_mode::JON_RPC, "rpc_access_data", req, res, r)) return r; if (!m_rpc_payment) @@ -3158,7 +3167,7 @@ namespace cryptonote RPC_TRACKER(rpc_access_account); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_ACCOUNT>(invoke_http_mode::JON, "rpc_access_account", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ACCESS_ACCOUNT>(invoke_http_mode::JON_RPC, "rpc_access_account", req, res, r)) return r; if (!m_rpc_payment) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 855ea854c..12b042c7e 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 0 +#define CORE_RPC_VERSION_MINOR 1 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -351,6 +351,7 @@ namespace cryptonote bool double_spend_seen; uint64_t block_height; uint64_t block_timestamp; + uint64_t received_timestamp; std::vector<uint64_t> output_indices; bool relayed; @@ -372,6 +373,7 @@ namespace cryptonote else { KV_SERIALIZE(relayed) + KV_SERIALIZE(received_timestamp) } END_KV_SERIALIZE_MAP() }; @@ -2556,22 +2558,21 @@ namespace cryptonote struct COMMAND_RPC_FLUSH_CACHE { - struct request_t + struct request_t: public rpc_request_base { bool bad_txs; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) KV_SERIALIZE_OPT(bad_txs, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; - struct response_t + struct response_t: public rpc_response_base { - std::string status; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(status) + KV_SERIALIZE_PARENT(rpc_response_base) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index d7e081af3..24800ff20 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -28,6 +28,7 @@ #include "daemon_handler.h" +#include <boost/uuid/nil_generator.hpp> // likely included by daemon_handler.h's includes, // but including here for clarity #include "cryptonote_core/cryptonote_core.h" @@ -288,10 +289,9 @@ namespace rpc return; } - cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, false, false, !relay) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, (relay ? relay_method::local : relay_method::none), false) || tvc.m_verifivation_failed) { if (tvc.m_verifivation_failed) { @@ -311,42 +311,42 @@ namespace rpc if (tvc.m_double_spend) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "double spend"; + res.error_details += "double spend"; } if (tvc.m_invalid_input) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "invalid input"; + res.error_details += "invalid input"; } if (tvc.m_invalid_output) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "invalid output"; + res.error_details += "invalid output"; } if (tvc.m_too_big) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "too big"; + res.error_details += "too big"; } if (tvc.m_overspend) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "overspend"; + res.error_details += "overspend"; } if (tvc.m_fee_too_low) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "fee too low"; + res.error_details += "fee too low"; } if (tvc.m_not_rct) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "tx is not ringct"; + res.error_details += "tx is not ringct"; } if (tvc.m_too_few_outputs) { if (!res.error_details.empty()) res.error_details += " and "; - res.error_details = "too few outputs"; + res.error_details += "too few outputs"; } if (res.error_details.empty()) { @@ -368,7 +368,7 @@ namespace rpc NOTIFY_NEW_TRANSACTIONS::request r; r.txs.push_back(tx_blob); - m_core.get_protocol()->relay_transactions(r, fake_context); + m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes res.status = Message::STATUS_OK; diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 0637db728..b363c27b2 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -59,6 +59,10 @@ namespace cryptonote { rpc_payment::client_info::client_info(): + previous_seed_height(0), + seed_height(0), + previous_seed_hash(crypto::null_hash), + seed_hash(crypto::null_hash), cookie(0), top(crypto::null_hash), previous_top(crypto::null_hash), diff --git a/src/rpc/rpc_version_str.cpp b/src/rpc/rpc_version_str.cpp new file mode 100644 index 000000000..c60cf4891 --- /dev/null +++ b/src/rpc/rpc_version_str.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "rpc_version_str.h" +#include "version.h" +#include <regex> + +namespace cryptonote +{ + +namespace rpc +{ + +// Expected format of Monero software version string: +// 1) Four numbers, one to two digits each, separated by periods +// 2) Optionally, one of the following suffixes: +// a) -release +// b) -<hash> where <hash> is exactly nine lowercase hex digits + +bool is_version_string_valid(const std::string& str) +{ + return std::regex_match(str, std::regex( + "^\\d{1,2}(\\.\\d{1,2}){3}(-(release|[0-9a-f]{9}))?$", + std::regex_constants::nosubs + )); +} + +} // namespace rpc + +} // namespace cryptonote diff --git a/src/rpc/rpc_version_str.h b/src/rpc/rpc_version_str.h new file mode 100644 index 000000000..930c807d2 --- /dev/null +++ b/src/rpc/rpc_version_str.h @@ -0,0 +1,43 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +#pragma once + +#include <string> + +namespace cryptonote +{ + +namespace rpc +{ + +bool is_version_string_valid(const std::string& str); + +} // namespace rpc + +} // namespace cryptonote diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index ea8f6f2f5..74e793669 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -121,7 +121,7 @@ typedef cryptonote::simple_wallet sw; #define SCOPED_WALLET_UNLOCK() SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return true;) -#define PRINT_USAGE(usage_help) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help; +#define PRINT_USAGE(usage_help_advanced) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help_advanced; #define LONG_PAYMENT_ID_SUPPORT_CHECK() \ do { \ @@ -196,7 +196,7 @@ namespace " account tag_description <tag_name> <description>"); const char* USAGE_ADDRESS("address [ new <label text with white spaces allowed> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed> | device [<index>]]"); const char* USAGE_INTEGRATED_ADDRESS("integrated_address [device] [<payment_id> | <address>]"); - const char* USAGE_ADDRESS_BOOK("address_book [(add ((<address> [pid <id>])|<integrated address>) [<description possibly with whitespaces>])|(delete <index>)]"); + const char* USAGE_ADDRESS_BOOK("address_book [(add (<address>|<integrated address>) [<description possibly with whitespaces>])|(delete <index>)]"); const char* USAGE_SET_VARIABLE("set <option> [<value>]"); const char* USAGE_GET_TX_KEY("get_tx_key <txid>"); const char* USAGE_SET_TX_KEY("set_tx_key <txid> <tx_key>"); @@ -214,7 +214,7 @@ namespace const char* USAGE_GET_TX_NOTE("get_tx_note <txid>"); const char* USAGE_GET_DESCRIPTION("get_description"); const char* USAGE_SET_DESCRIPTION("set_description [free text note]"); - const char* USAGE_SIGN("sign <filename>"); + const char* USAGE_SIGN("sign [<account_index>,<address_index>] <filename>"); const char* USAGE_VERIFY("verify <filename> <address> <signature>"); const char* USAGE_EXPORT_KEY_IMAGES("export_key_images [all] <filename>"); const char* USAGE_IMPORT_KEY_IMAGES("import_key_images <filename>"); @@ -268,7 +268,8 @@ namespace const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc"); const char* USAGE_STOP_MINING_FOR_RPC("stop_mining_for_rpc"); const char* USAGE_VERSION("version"); - const char* USAGE_HELP("help [<command>]"); + const char* USAGE_HELP_ADVANCED("help_advanced [<command>]"); + const char* USAGE_HELP("help"); std::string input_line(const std::string& prompt, bool yesno = false) { @@ -912,16 +913,6 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { LONG_PAYMENT_ID_SUPPORT_CHECK(); - - crypto::hash payment_id; - if (args.size() > 0) - { - PRINT_USAGE(USAGE_PAYMENT_ID); - return true; - } - payment_id = crypto::rand<crypto::hash>(); - success_msg_writer() << tr("Random payment ID: ") << payment_id; - return true; } bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -2315,7 +2306,7 @@ bool simple_wallet::on_unknown_command(const std::vector<std::string> &args) { if (args[0] == "exit" || args[0] == "q") // backward compat return false; - fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help'")) % args.front(); + fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help_advanced'")) % args.front(); return true; } @@ -3044,13 +3035,37 @@ bool simple_wallet::set_export_format(const std::vector<std::string> &args/* = s bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + message_writer() << ""; + message_writer() << tr("Commands:"); + message_writer() << ""; + message_writer() << tr("\"welcome\" - Read welcome message."); + message_writer() << tr("\"donate <amount>\" - Donate XMR to the development team."); + message_writer() << tr("\"balance\" - Show balance."); + message_writer() << tr("\"address new\" - Create new subaddress."); + message_writer() << tr("\"address all\" - Show all addresses."); + message_writer() << tr("\"transfer <address> <amount>\" - Send XMR to an address."); + message_writer() << tr("\"show_transfers [in|out|pending|failed|pool]\" - Show transactions."); + message_writer() << tr("\"sweep_all <address>\" - Send whole balance to another wallet."); + message_writer() << tr("\"seed\" - Show secret 25 words that can be used to recover this wallet."); + message_writer() << tr("\"refresh\" - Synchronize wallet with the Monero network."); + message_writer() << tr("\"status\" - Check current status of wallet."); + message_writer() << tr("\"version\" - Check software version."); + message_writer() << tr("\"help_advanced\" - Show list with more available commands."); + message_writer() << tr("\"save\" - Save wallet."); + message_writer() << tr("\"exit\" - Exit wallet."); + message_writer() << ""; + return true; +} + +bool simple_wallet::help_advanced(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ if(args.empty()) { success_msg_writer() << get_commands_str(); } else if ((args.size() == 2) && (args.front() == "mms")) { - // Little hack to be able to do "help mms <subcommand>" + // Little hack to be able to do "help_advanced mms <subcommand>" std::vector<std::string> mms_args(1, args.front() + " " + args.back()); success_msg_writer() << get_command_usage(mms_args); } @@ -3262,7 +3277,9 @@ simple_wallet::simple_wallet() "auto-mine-for-rpc-payment-threshold <float>\n " " Whether to automatically start mining for RPC payment if the daemon requires it.\n" "credits-target <unsigned int>\n" - " The RPC payment credits balance to target (0 for default).")); + " The RPC payment credits balance to target (0 for default).\n " + "inactivity-lock-timeout <unsigned int>\n " + " How many seconds to wait before locking the wallet (0 to disable).")); m_cmd_binder.set_handler("encrypted_seed", boost::bind(&simple_wallet::on_command, this, &simple_wallet::encrypted_seed, _1), tr("Display the encrypted Electrum-style mnemonic seed.")); @@ -3356,7 +3373,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("sign", boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign, _1), tr(USAGE_SIGN), - tr("Sign the contents of a file.")); + tr("Sign the contents of a file with the given subaddress (or the main address if not specified)")); m_cmd_binder.set_handler("verify", boost::bind(&simple_wallet::on_command, this, &simple_wallet::verify, _1), tr(USAGE_VERIFY), @@ -3439,7 +3456,7 @@ simple_wallet::simple_wallet() "<subcommand> is one of:\n" " init, info, signer, list, next, sync, transfer, delete, send, receive, export, note, show, set, help\n" " send_signer_config, start_auto_config, stop_auto_config, auto_config\n" - "Get help about a subcommand with: help mms <subcommand>, or mms help <subcommand>")); + "Get help about a subcommand with: help_advanced mms <subcommand>")); m_cmd_binder.set_handler("mms init", boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_INIT), @@ -3589,10 +3606,14 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::stop_mining_for_rpc, this, _1), tr(USAGE_STOP_MINING_FOR_RPC), tr("Stop mining to pay for RPC access")); + m_cmd_binder.set_handler("help_advanced", + boost::bind(&simple_wallet::on_command, this, &simple_wallet::help_advanced, _1), + tr(USAGE_HELP_ADVANCED), + tr("Show the help section or the documentation about a <command>.")); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::on_command, this, &simple_wallet::help, _1), tr(USAGE_HELP), - tr("Show the help section or the documentation about a <command>.")); + tr("Show simplified list of available commands.")); m_cmd_binder.set_unknown_command_handler(boost::bind(&simple_wallet::on_command, this, &simple_wallet::on_unknown_command, _1)); m_cmd_binder.set_empty_command_handler(boost::bind(&simple_wallet::on_empty_command, this)); m_cmd_binder.set_cancel_handler(boost::bind(&simple_wallet::on_cancelled_command, this)); @@ -4356,7 +4377,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (std::cin.eof() || !command_line::is_yes(confirm)) CHECK_AND_ASSERT_MES(false, false, tr("account creation aborted")); - m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()); + m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height() > 0 ? m_wallet->estimate_blockchain_height() - 1 : 0); m_wallet->explicit_refresh_from_block_height(true); m_restore_height = m_wallet->get_refresh_from_block_height(); } @@ -4744,8 +4765,9 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr "**********************************************************************\n" << tr("Your wallet has been generated!\n" "To start synchronizing with the daemon, use the \"refresh\" command.\n" - "Use the \"help\" command to see the list of available commands.\n" - "Use \"help <command>\" to see a command's documentation.\n" + "Use the \"help\" command to see a simplified list of available commands.\n" + "Use the \"help_advanced\" command to see an advanced list of available commands.\n" + "Use \"help_advanced <command>\" to see a command's documentation.\n" "Always use the \"exit\" command when closing monero-wallet-cli to save \n" "your current session's state. Otherwise, you might need to synchronize \n" "your wallet again (your wallet keys are NOT at risk in any case).\n") @@ -5004,8 +5026,9 @@ boost::optional<epee::wipeable_string> simple_wallet::open_wallet(const boost::p } success_msg_writer() << "**********************************************************************\n" << - tr("Use the \"help\" command to see the list of available commands.\n") << - tr("Use \"help <command>\" to see a command's documentation.\n") << + tr("Use the \"help\" command to see a simplified list of available commands.\n") << + tr("Use the \"help_advanced\" command to see an advanced list of available commands.\n") << + tr("Use \"help_advanced <command>\" to see a command's documentation.\n") << "**********************************************************************"; return password; } @@ -6195,7 +6218,8 @@ void simple_wallet::check_for_inactivity_lock(bool user) } while (1) { - tools::msg_writer() << tr("Locked due to inactivity. The wallet password is required to unlock the console."); + const char *inactivity_msg = user ? "" : tr("Locked due to inactivity."); + tools::msg_writer() << inactivity_msg << (inactivity_msg[0] ? " " : "") << tr("The wallet password is required to unlock the console."); try { if (get_and_verify_password()) @@ -6293,13 +6317,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { LONG_PAYMENT_ID_SUPPORT_CHECK(); - - std::string extra_nonce; - set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - local_args.pop_back(); - payment_id_seen = true; - message_writer() << tr("Warning: Unencrypted payment IDs will harm your privacy: ask the recipient to use subaddresses instead"); } if(!r) { @@ -6408,8 +6425,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) { LONG_PAYMENT_ID_SUPPORT_CHECK(); - set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); - message_writer() << tr("Warning: Unencrypted payment IDs will harm your privacy: ask the recipient to use subaddresses instead"); } else { @@ -6945,11 +6960,6 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st if(r) { LONG_PAYMENT_ID_SUPPORT_CHECK(); - - std::string extra_nonce; - set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - payment_id_seen = true; } if(!r && local_args.size() == 3) @@ -7191,7 +7201,6 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (tools::wallet2::parse_long_payment_id(local_args.back(), payment_id)) { LONG_PAYMENT_ID_SUPPORT_CHECK(); - set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); } else { @@ -9410,30 +9419,7 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v fail_msg_writer() << tr("failed to parse address"); return true; } - crypto::hash payment_id = crypto::null_hash; size_t description_start = 2; - if (info.has_payment_id) - { - memcpy(payment_id.data, info.payment_id.data, 8); - } - else if (!info.has_payment_id && args.size() >= 4 && args[2] == "pid") - { - if (tools::wallet2::parse_long_payment_id(args[3], payment_id)) - { - LONG_PAYMENT_ID_SUPPORT_CHECK(); - description_start += 2; - } - else if (tools::wallet2::parse_short_payment_id(args[3], info.payment_id)) - { - fail_msg_writer() << tr("Short payment IDs are to be used within an integrated address only"); - return true; - } - else - { - fail_msg_writer() << tr("failed to parse payment ID"); - return true; - } - } std::string description; for (size_t i = description_start; i < args.size(); ++i) { @@ -9441,7 +9427,7 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v description += " "; description += args[i]; } - m_wallet->add_address_book_row(info.address, payment_id, description, info.is_subaddress); + m_wallet->add_address_book_row(info.address, info.has_payment_id ? &info.payment_id : NULL, description, info.is_subaddress); } else { @@ -9463,8 +9449,12 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v for (size_t i = 0; i < address_book.size(); ++i) { auto& row = address_book[i]; success_msg_writer() << tr("Index: ") << i; - success_msg_writer() << tr("Address: ") << get_account_address_as_str(m_wallet->nettype(), row.m_is_subaddress, row.m_address); - success_msg_writer() << tr("Payment ID: ") << row.m_payment_id << " (OBSOLETE)"; + std::string address; + if (row.m_has_payment_id) + address = cryptonote::get_account_integrated_address_as_str(m_wallet->nettype(), row.m_address, row.m_payment_id); + else + address = get_account_address_as_str(m_wallet->nettype(), row.m_is_subaddress, row.m_address); + success_msg_writer() << tr("Address: ") << address; success_msg_writer() << tr("Description: ") << row.m_description << "\n"; } } @@ -9616,7 +9606,7 @@ bool simple_wallet::sign(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } - if (args.size() != 1) + if (args.size() != 1 && args.size() != 2) { PRINT_USAGE(USAGE_SIGN); return true; @@ -9632,7 +9622,20 @@ bool simple_wallet::sign(const std::vector<std::string> &args) return true; } - std::string filename = args[0]; + subaddress_index index{0, 0}; + if (args.size() == 2) + { + unsigned int a, b; + if (sscanf(args[0].c_str(), "%u,%u", &a, &b) != 2) + { + fail_msg_writer() << tr("Invalid subaddress index format"); + return true; + } + index.major = a; + index.minor = b; + } + + const std::string &filename = args.back(); std::string data; bool r = m_wallet->load_from_file(filename, data); if (!r) @@ -9643,7 +9646,7 @@ bool simple_wallet::sign(const std::vector<std::string> &args) SCOPED_WALLET_UNLOCK(); - std::string signature = m_wallet->sign(data); + std::string signature = m_wallet->sign(data, index); success_msg_writer() << signature; return true; } @@ -11016,7 +11019,7 @@ void simple_wallet::mms_help(const std::vector<std::string> &args) { if (args.size() > 1) { - fail_msg_writer() << tr("Usage: mms help [<subcommand>]"); + fail_msg_writer() << tr("Usage: help_advanced mms [<subcommand>]"); return; } std::vector<std::string> help_args; diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 75bd893d5..e401f5fda 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -154,6 +154,7 @@ namespace cryptonote bool set_persistent_rpc_client_id(const std::vector<std::string> &args = std::vector<std::string>()); bool set_auto_mine_for_rpc_payment_threshold(const std::vector<std::string> &args = std::vector<std::string>()); bool set_credits_target(const std::vector<std::string> &args = std::vector<std::string>()); + bool help_advanced(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index 3be1a6f6b..a0a166a93 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -77,42 +77,43 @@ target_link_libraries(wallet PRIVATE ${EXTRA_LIBRARIES}) -set(wallet_rpc_sources - wallet_rpc_server.cpp) +if(NOT IOS) + set(wallet_rpc_sources + wallet_rpc_server.cpp) -set(wallet_rpc_headers) + set(wallet_rpc_headers) -set(wallet_rpc_private_headers - wallet_rpc_server.h) + set(wallet_rpc_private_headers + wallet_rpc_server.h) -monero_private_headers(wallet_rpc_server - ${wallet_rpc_private_headers}) -monero_add_executable(wallet_rpc_server - ${wallet_rpc_sources} - ${wallet_rpc_headers} - ${wallet_rpc_private_headers}) - -target_link_libraries(wallet_rpc_server - PRIVATE - wallet - rpc_base - cryptonote_core - cncrypto - common - version - daemonizer - ${EPEE_READLINE} - ${Boost_CHRONO_LIBRARY} - ${Boost_PROGRAM_OPTIONS_LIBRARY} - ${Boost_FILESYSTEM_LIBRARY} - ${Boost_THREAD_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT} - ${EXTRA_LIBRARIES}) -set_property(TARGET wallet_rpc_server - PROPERTY - OUTPUT_NAME "monero-wallet-rpc") -install(TARGETS wallet_rpc_server DESTINATION bin) + monero_private_headers(wallet_rpc_server + ${wallet_rpc_private_headers}) + monero_add_executable(wallet_rpc_server + ${wallet_rpc_sources} + ${wallet_rpc_headers} + ${wallet_rpc_private_headers}) + target_link_libraries(wallet_rpc_server + PRIVATE + wallet + rpc_base + cryptonote_core + cncrypto + common + version + daemonizer + ${EPEE_READLINE} + ${Boost_CHRONO_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + set_property(TARGET wallet_rpc_server + PROPERTY + OUTPUT_NAME "monero-wallet-rpc") + install(TARGETS wallet_rpc_server DESTINATION bin) +endif() # build and install libwallet_merged only if we building for GUI if (BUILD_GUI_DEPS) diff --git a/src/wallet/api/address_book.cpp b/src/wallet/api/address_book.cpp index 7be78bba7..005ddf7ee 100644 --- a/src/wallet/api/address_book.cpp +++ b/src/wallet/api/address_book.cpp @@ -55,37 +55,14 @@ bool AddressBookImpl::addRow(const std::string &dst_addr , const std::string &pa return false; } - crypto::hash payment_id = crypto::null_hash; - bool has_long_pid = (payment_id_str.empty())? false : tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - - // Short payment id provided - if(payment_id_str.length() == 16) { - m_errorString = tr("Invalid payment ID. Short payment ID should only be used in an integrated address"); - m_errorCode = Invalid_Payment_Id; - return false; - } - - // long payment id provided but not valid - if(!payment_id_str.empty() && !has_long_pid) { - m_errorString = tr("Invalid payment ID"); - m_errorCode = Invalid_Payment_Id; - return false; - } - - // integrated + long payment id provided - if(has_long_pid && info.has_payment_id) { - m_errorString = tr("Integrated address and long payment ID can't be used at the same time"); + if (!payment_id_str.empty()) + { + m_errorString = tr("Payment ID supplied: this is obsolete"); m_errorCode = Invalid_Payment_Id; return false; } - // Pad short pid with zeros - if (info.has_payment_id) - { - memcpy(payment_id.data, info.payment_id.data, 8); - } - - bool r = m_wallet->m_wallet->add_address_book_row(info.address,payment_id,description,info.is_subaddress); + bool r = m_wallet->m_wallet->add_address_book_row(info.address, info.has_payment_id ? &info.payment_id : NULL,description,info.is_subaddress); if (r) refresh(); else @@ -104,19 +81,12 @@ void AddressBookImpl::refresh() for (size_t i = 0; i < rows.size(); ++i) { tools::wallet2::address_book_row * row = &rows.at(i); - std::string payment_id = (row->m_payment_id == crypto::null_hash)? "" : epee::string_tools::pod_to_hex(row->m_payment_id); - std::string address = cryptonote::get_account_address_as_str(m_wallet->m_wallet->nettype(), row->m_is_subaddress, row->m_address); - // convert the zero padded short payment id to integrated address - if (!row->m_is_subaddress && payment_id.length() > 16 && payment_id.substr(16).find_first_not_of('0') == std::string::npos) { - payment_id = payment_id.substr(0,16); - crypto::hash8 payment_id_short; - if(tools::wallet2::parse_short_payment_id(payment_id, payment_id_short)) { - address = cryptonote::get_account_integrated_address_as_str(m_wallet->m_wallet->nettype(), row->m_address, payment_id_short); - // Don't show payment id when integrated address is used - payment_id = ""; - } - } - AddressBookRow * abr = new AddressBookRow(i, address, payment_id, row->m_description); + std::string address; + if (row->m_has_payment_id) + address = cryptonote::get_account_integrated_address_as_str(m_wallet->m_wallet->nettype(), row->m_address, row->m_payment_id); + else + address = get_account_address_as_str(m_wallet->m_wallet->nettype(), row->m_is_subaddress, row->m_address); + AddressBookRow * abr = new AddressBookRow(i, address, "", row->m_description); m_rows.push_back(abr); } diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 15ea26044..f3698b599 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -36,7 +36,7 @@ do { \ CHECK_AND_ASSERT_MES(error.code == 0, error.message, error.message); \ handle_payment_changes(res, std::integral_constant<bool, HasCredits<decltype(res)>::Has>()); \ - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); \ + CHECK_AND_ASSERT_MES(r, std::string("Failed to connect to daemon"), "Failed to connect to daemon"); \ /* empty string -> not connection */ \ CHECK_AND_ASSERT_MES(!res.status.empty(), res.status, "No connection to daemon"); \ CHECK_AND_ASSERT_MES(res.status != CORE_RPC_STATUS_BUSY, res.status, "Daemon busy"); \ @@ -78,6 +78,10 @@ void NodeRPCProxy::invalidate() m_rpc_payment_seed_hash = crypto::null_hash; m_rpc_payment_next_seed_hash = crypto::null_hash; m_height_time = 0; + m_rpc_payment_diff = 0; + m_rpc_payment_credits_per_hash_found = 0; + m_rpc_payment_height = 0; + m_rpc_payment_cookie = 0; } boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index e60c6b7e1..870aa65ac 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1038,10 +1038,15 @@ uint64_t gamma_picker::pick() return first_rct + crypto::rand_idx(n_rct); }; +boost::mutex wallet_keys_unlocker::lockers_lock; +unsigned int wallet_keys_unlocker::lockers = 0; wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password): w(w), locked(password != boost::none) { + boost::lock_guard<boost::mutex> lock(lockers_lock); + if (lockers++ > 0) + locked = false; if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt || w.watch_only()) { locked = false; @@ -1056,6 +1061,9 @@ wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, bool locked, const epee:: w(w), locked(locked) { + boost::lock_guard<boost::mutex> lock(lockers_lock); + if (lockers++ > 0) + locked = false; if (!locked) return; w.generate_chacha_key_from_password(password, key); @@ -1064,9 +1072,19 @@ wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, bool locked, const epee:: wallet_keys_unlocker::~wallet_keys_unlocker() { - if (!locked) - return; - try { w.encrypt_keys(key); } + try + { + boost::lock_guard<boost::mutex> lock(lockers_lock); + if (lockers == 0) + { + MERROR("There are no lockers in wallet_keys_unlocker dtor"); + return; + } + --lockers; + if (!locked) + return; + w.encrypt_keys(key); + } catch (...) { MERROR("Failed to re-encrypt wallet keys"); @@ -3133,11 +3151,12 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, } -bool wallet2::add_address_book_row(const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress) +bool wallet2::add_address_book_row(const cryptonote::account_public_address &address, const crypto::hash8 *payment_id, const std::string &description, bool is_subaddress) { wallet2::address_book_row a; a.m_address = address; - a.m_payment_id = payment_id; + a.m_has_payment_id = !!payment_id; + a.m_payment_id = payment_id ? *payment_id : crypto::null_hash8; a.m_description = description; a.m_is_subaddress = is_subaddress; @@ -3148,11 +3167,12 @@ bool wallet2::add_address_book_row(const cryptonote::account_public_address &add return false; } -bool wallet2::set_address_book_row(size_t row_id, const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress) +bool wallet2::set_address_book_row(size_t row_id, const cryptonote::account_public_address &address, const crypto::hash8 *payment_id, const std::string &description, bool is_subaddress) { wallet2::address_book_row a; a.m_address = address; - a.m_payment_id = payment_id; + a.m_has_payment_id = !!payment_id; + a.m_payment_id = payment_id ? *payment_id : crypto::null_hash8; a.m_description = description; a.m_is_subaddress = is_subaddress; @@ -5394,6 +5414,7 @@ bool wallet2::check_connection(uint32_t *version, bool *ssl, uint32_t timeout) void wallet2::set_offline(bool offline) { m_offline = offline; + m_node_rpc_proxy.set_offline(offline); m_http_client.set_auto_connect(!offline); if (offline) { @@ -7555,6 +7576,8 @@ bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) bool wallet2::lock_keys_file() { + if (m_wallet_file.empty()) + return true; if (m_keys_file_locker) { MDEBUG(m_keys_file << " is already locked."); @@ -7566,6 +7589,8 @@ bool wallet2::lock_keys_file() bool wallet2::unlock_keys_file() { + if (m_wallet_file.empty()) + return true; if (!m_keys_file_locker) { MDEBUG(m_keys_file << " is already unlocked."); @@ -7577,6 +7602,8 @@ bool wallet2::unlock_keys_file() bool wallet2::is_keys_file_locked() const { + if (m_wallet_file.empty()) + return false; return m_keys_file_locker->locked(); } @@ -11802,13 +11829,27 @@ void wallet2::set_account_tag_description(const std::string& tag, const std::str m_account_tags.first[tag] = description; } -std::string wallet2::sign(const std::string &data) const +std::string wallet2::sign(const std::string &data, cryptonote::subaddress_index index) const { crypto::hash hash; crypto::cn_fast_hash(data.data(), data.size(), hash); const cryptonote::account_keys &keys = m_account.get_keys(); crypto::signature signature; - crypto::generate_signature(hash, keys.m_account_address.m_spend_public_key, keys.m_spend_secret_key, signature); + crypto::secret_key skey; + crypto::public_key pkey; + if (index.is_zero()) + { + skey = keys.m_spend_secret_key; + pkey = keys.m_account_address.m_spend_public_key; + } + else + { + skey = keys.m_spend_secret_key; + crypto::secret_key m = m_account.get_device().get_subaddress_secret_key(keys.m_view_secret_key, index); + sc_add((unsigned char*)&skey, (unsigned char*)&m, (unsigned char*)&skey); + secret_key_to_public_key(skey, pkey); + } + crypto::generate_signature(hash, pkey, skey, signature); return std::string("SigV1") + tools::base58::encode(std::string((const char *)&signature, sizeof(signature))); } @@ -13601,4 +13642,22 @@ std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only) std::copy(res.gray.begin(), res.gray.end(), std::back_inserter(nodes)); return nodes; } +//---------------------------------------------------------------------------------------------------- +std::pair<size_t, uint64_t> wallet2::estimate_tx_size_and_weight(bool use_rct, int n_inputs, int ring_size, int n_outputs, size_t extra_size) +{ + THROW_WALLET_EXCEPTION_IF(n_inputs <= 0, tools::error::wallet_internal_error, "Invalid n_inputs"); + THROW_WALLET_EXCEPTION_IF(n_outputs < 0, tools::error::wallet_internal_error, "Invalid n_outputs"); + THROW_WALLET_EXCEPTION_IF(ring_size < 0, tools::error::wallet_internal_error, "Invalid ring size"); + + if (ring_size == 0) + ring_size = get_min_ring_size(); + if (n_outputs == 1) + n_outputs = 2; // extra dummy output + + const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); + size_t size = estimate_tx_size(use_rct, n_inputs, ring_size - 1, n_outputs, extra_size, bulletproof); + uint64_t weight = estimate_tx_weight(use_rct, n_inputs, ring_size - 1, n_outputs, extra_size, bulletproof); + return std::make_pair(size, weight); +} +//---------------------------------------------------------------------------------------------------- } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index c86315f7c..b17fe6f3a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -122,6 +122,8 @@ private: wallet2 &w; bool locked; crypto::chacha_key key; + static boost::mutex lockers_lock; + static unsigned int lockers; }; class i_wallet2_callback @@ -544,9 +546,10 @@ private: struct address_book_row { cryptonote::account_public_address m_address; - crypto::hash m_payment_id; + crypto::hash8 m_payment_id; std::string m_description; bool m_is_subaddress; + bool m_has_payment_id; }; struct reserve_proof_entry @@ -1123,8 +1126,8 @@ private: * \brief GUI Address book get/store */ std::vector<address_book_row> get_address_book() const { return m_address_book; } - bool add_address_book_row(const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress); - bool set_address_book_row(size_t row_id, const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress); + bool add_address_book_row(const cryptonote::account_public_address &address, const crypto::hash8 *payment_id, const std::string &description, bool is_subaddress); + bool set_address_book_row(size_t row_id, const cryptonote::account_public_address &address, const crypto::hash8 *payment_id, const std::string &description, bool is_subaddress); bool delete_address_book_row(std::size_t row_id); uint64_t get_num_rct_outputs(); @@ -1182,7 +1185,7 @@ private: */ void set_account_tag_description(const std::string& tag, const std::string& description); - std::string sign(const std::string &data) const; + std::string sign(const std::string &data, cryptonote::subaddress_index index = {0, 0}) const; bool verify(const std::string &data, const cryptonote::account_public_address &address, const std::string &signature) const; /*! @@ -1252,6 +1255,8 @@ private: bool is_unattended() const { return m_unattended; } + std::pair<size_t, uint64_t> estimate_tx_size_and_weight(bool use_rct, int n_inputs, int ring_size, int n_outputs, size_t extra_size); + bool get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &hashing_blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie); bool daemon_requires_payment(); bool make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credits, uint64_t &balance); @@ -1630,7 +1635,7 @@ BOOST_CLASS_VERSION(tools::wallet2::payment_details, 4) BOOST_CLASS_VERSION(tools::wallet2::pool_payment_details, 1) BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8) BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) -BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) +BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 18) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) @@ -1950,7 +1955,26 @@ namespace boost inline void serialize(Archive& a, tools::wallet2::address_book_row& x, const boost::serialization::version_type ver) { a & x.m_address; - a & x.m_payment_id; + if (ver < 18) + { + crypto::hash payment_id; + a & payment_id; + x.m_has_payment_id = !(payment_id == crypto::null_hash); + if (x.m_has_payment_id) + { + bool is_long = false; + for (int i = 8; i < 32; ++i) + is_long |= payment_id.data[i]; + if (is_long) + { + MWARNING("Long payment ID ignored on address book load"); + x.m_payment_id = crypto::null_hash8; + x.m_has_payment_id = false; + } + else + memcpy(x.m_payment_id.data, payment_id.data, 8); + } + } a & x.m_description; if (ver < 17) { @@ -1958,6 +1982,11 @@ namespace boost return; } a & x.m_is_subaddress; + if (ver < 18) + return; + a & x.m_has_payment_id; + if (x.m_has_payment_id) + a & x.m_payment_id; } template <class Archive> diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index de501f056..d282d7cb2 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -550,9 +550,29 @@ namespace tools if (!m_wallet) return not_open(er); try { - m_wallet->add_subaddress(req.account_index, req.label); - res.address_index = m_wallet->get_num_subaddresses(req.account_index) - 1; - res.address = m_wallet->get_subaddress_as_str({req.account_index, res.address_index}); + if (req.count < 1 || req.count > 64) { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Count must be between 1 and 64."; + return false; + } + + std::vector<std::string> addresses; + std::vector<uint32_t> address_indices; + + addresses.reserve(req.count); + address_indices.reserve(req.count); + + for (uint32_t i = 0; i < req.count; i++) { + m_wallet->add_subaddress(req.account_index, req.label); + uint32_t new_address_index = m_wallet->get_num_subaddresses(req.account_index) - 1; + address_indices.push_back(new_address_index); + addresses.push_back(m_wallet->get_subaddress_as_str({req.account_index, new_address_index})); + } + + res.address = addresses[0]; + res.address_index = address_indices[0]; + res.addresses = addresses; + res.address_indices = address_indices; } catch (const std::exception& e) { @@ -843,7 +863,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ template<typename Ts, typename Tu> bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er) { for (const auto & ptx : ptx_vector) @@ -858,6 +878,7 @@ namespace tools // Compute amount leaving wallet in tx. By convention dests does not include change outputs fill(amount, total_amount(ptx)); fill(fee, ptx.fee); + fill(weight, cryptonote::get_transaction_weight(ptx.tx)); } if (m_wallet->multisig()) @@ -943,7 +964,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); } catch (const std::exception& e) @@ -989,7 +1010,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -1353,7 +1374,7 @@ namespace tools { std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -1400,7 +1421,7 @@ namespace tools uint32_t priority = m_wallet->adjust_priority(req.priority); std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -1475,7 +1496,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); } catch (const std::exception& e) @@ -1943,7 +1964,7 @@ namespace tools return false; } - res.signature = m_wallet->sign(req.data); + res.signature = m_wallet->sign(req.data, {req.account_index, req.address_index}); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -2725,7 +2746,14 @@ namespace tools { uint64_t idx = 0; for (const auto &entry: ab) - res.entries.push_back(wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::entry{idx++, get_account_address_as_str(m_wallet->nettype(), entry.m_is_subaddress, entry.m_address), epee::string_tools::pod_to_hex(entry.m_payment_id), entry.m_description}); + { + std::string address; + if (entry.m_has_payment_id) + address = cryptonote::get_account_integrated_address_as_str(m_wallet->nettype(), entry.m_address, entry.m_payment_id); + else + address = get_account_address_as_str(m_wallet->nettype(), entry.m_is_subaddress, entry.m_address); + res.entries.push_back(wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::entry{idx++, address, entry.m_description}); + } } else { @@ -2738,7 +2766,12 @@ namespace tools return false; } const auto &entry = ab[idx]; - res.entries.push_back(wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::entry{idx, get_account_address_as_str(m_wallet->nettype(), entry.m_is_subaddress, entry.m_address), epee::string_tools::pod_to_hex(entry.m_payment_id), entry.m_description}); + std::string address; + if (entry.m_has_payment_id) + address = cryptonote::get_account_integrated_address_as_str(m_wallet->nettype(), entry.m_address, entry.m_payment_id); + else + address = get_account_address_as_str(m_wallet->nettype(), entry.m_is_subaddress, entry.m_address); + res.entries.push_back(wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::entry{idx, address, entry.m_description}); } } return true; @@ -2755,7 +2788,6 @@ namespace tools } cryptonote::address_parse_info info; - crypto::hash payment_id = crypto::null_hash; er.message = ""; if(!get_account_address_from_str_or_url(info, m_wallet->nettype(), req.address, [&er](const std::string &url, const std::vector<std::string> &addresses, bool dnssec_valid)->std::string { @@ -2777,39 +2809,7 @@ namespace tools er.message = std::string("WALLET_RPC_ERROR_CODE_WRONG_ADDRESS: ") + req.address; return false; } - if (info.has_payment_id) - { - memcpy(payment_id.data, info.payment_id.data, 8); - memset(payment_id.data + 8, 0, 24); - } - if (!req.payment_id.empty()) - { - if (info.has_payment_id) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Separate payment ID given with integrated address"; - return false; - } - - crypto::hash long_payment_id; - - if (!wallet2::parse_long_payment_id(req.payment_id, payment_id)) - { - if (!wallet2::parse_short_payment_id(req.payment_id, info.payment_id)) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 64 character string"; - return false; - } - else - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: standalone short payment IDs are forbidden, they must be part of an integrated address"; - return false; - } - } - } - if (!m_wallet->add_address_book_row(info.address, payment_id, req.description, info.is_subaddress)) + if (!m_wallet->add_address_book_row(info.address, info.has_payment_id ? &info.payment_id : NULL, req.description, info.is_subaddress)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Failed to add address book entry"; @@ -2840,7 +2840,6 @@ namespace tools tools::wallet2::address_book_row entry = ab[req.index]; cryptonote::address_parse_info info; - crypto::hash payment_id = crypto::null_hash; if (req.set_address) { er.message = ""; @@ -2867,52 +2866,13 @@ namespace tools entry.m_address = info.address; entry.m_is_subaddress = info.is_subaddress; if (info.has_payment_id) - { - memcpy(entry.m_payment_id.data, info.payment_id.data, 8); - memset(entry.m_payment_id.data + 8, 0, 24); - } - } - - if (req.set_payment_id) - { - if (req.payment_id.empty()) - { - payment_id = crypto::null_hash; - } - else - { - if (req.set_address && info.has_payment_id) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Separate payment ID given with integrated address"; - return false; - } - - if (!wallet2::parse_long_payment_id(req.payment_id, payment_id)) - { - crypto::hash8 spid; - if (!wallet2::parse_short_payment_id(req.payment_id, spid)) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 64 character string"; - return false; - } - else - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: standalone short payment IDs are forbidden, they must be part of an integrated address"; - return false; - } - } - } - - entry.m_payment_id = payment_id; + entry.m_payment_id = info.payment_id; } if (req.set_description) entry.m_description = req.description; - if (!m_wallet->set_address_book_row(req.index, entry.m_address, entry.m_payment_id, entry.m_description, entry.m_is_subaddress)) + if (!m_wallet->set_address_book_row(req.index, entry.m_address, req.set_address && entry.m_has_payment_id ? &entry.m_payment_id : NULL, entry.m_description, entry.m_is_subaddress)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Failed to edit address book entry"; @@ -4291,6 +4251,25 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_estimate_tx_size_and_weight(const wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::request& req, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + size_t extra_size = 34 /* pubkey */ + 10 /* encrypted payment id */; // typical makeup + const std::pair<size_t, uint64_t> sw = m_wallet->estimate_tx_size_and_weight(req.rct, req.n_inputs, req.ring_size, req.n_outputs, extra_size); + res.size = sw.first; + res.weight = sw.second; + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to determine size and weight"; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx) { res.version = WALLET_RPC_VERSION; diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index b2b5e7116..89bf3a924 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -154,6 +154,7 @@ namespace tools MAP_JON_RPC_WE("set_daemon", on_set_daemon, wallet_rpc::COMMAND_RPC_SET_DAEMON) MAP_JON_RPC_WE("set_log_level", on_set_log_level, wallet_rpc::COMMAND_RPC_SET_LOG_LEVEL) MAP_JON_RPC_WE("set_log_categories", on_set_log_categories, wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES) + MAP_JON_RPC_WE("estimate_tx_size_and_weight", on_estimate_tx_size_and_weight, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT) MAP_JON_RPC_WE("get_version", on_get_version, wallet_rpc::COMMAND_RPC_GET_VERSION) END_JSON_RPC_MAP() END_URI_MAP2() @@ -240,6 +241,7 @@ namespace tools bool on_set_daemon(const wallet_rpc::COMMAND_RPC_SET_DAEMON::request& req, wallet_rpc::COMMAND_RPC_SET_DAEMON::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_set_log_level(const wallet_rpc::COMMAND_RPC_SET_LOG_LEVEL::request& req, wallet_rpc::COMMAND_RPC_SET_LOG_LEVEL::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_set_log_categories(const wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES::request& req, wallet_rpc::COMMAND_RPC_SET_LOG_CATEGORIES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_estimate_tx_size_and_weight(const wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::request& req, wallet_rpc::COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); //json rpc v2 @@ -255,7 +257,7 @@ namespace tools template<typename Ts, typename Tu> bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er); bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 0c86f404d..d6735e117 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 16 +#define WALLET_RPC_VERSION_MINOR 17 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -182,11 +182,13 @@ namespace wallet_rpc { struct request_t { - uint32_t account_index; + uint32_t account_index; + uint32_t count; std::string label; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) + KV_SERIALIZE_OPT(count, 1U) KV_SERIALIZE(label) END_KV_SERIALIZE_MAP() }; @@ -194,12 +196,16 @@ namespace wallet_rpc struct response_t { - std::string address; - uint32_t address_index; + std::string address; + uint32_t address_index; + std::vector<std::string> addresses; + std::vector<uint32_t> address_indices; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) KV_SERIALIZE(address_index) + KV_SERIALIZE(addresses) + KV_SERIALIZE(address_indices) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -484,6 +490,7 @@ namespace wallet_rpc std::string tx_key; uint64_t amount; uint64_t fee; + uint64_t weight; std::string tx_blob; std::string tx_metadata; std::string multisig_txset; @@ -494,6 +501,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_key) KV_SERIALIZE(amount) KV_SERIALIZE(fee) + KV_SERIALIZE(weight) KV_SERIALIZE(tx_blob) KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) @@ -550,6 +558,7 @@ namespace wallet_rpc std::list<std::string> tx_key_list; std::list<uint64_t> amount_list; std::list<uint64_t> fee_list; + std::list<uint64_t> weight_list; std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; @@ -560,6 +569,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_key_list) KV_SERIALIZE(amount_list) KV_SERIALIZE(fee_list) + KV_SERIALIZE(weight_list) KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) @@ -723,6 +733,7 @@ namespace wallet_rpc std::list<std::string> tx_key_list; std::list<uint64_t> amount_list; std::list<uint64_t> fee_list; + std::list<uint64_t> weight_list; std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; @@ -733,6 +744,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_key_list) KV_SERIALIZE(amount_list) KV_SERIALIZE(fee_list) + KV_SERIALIZE(weight_list) KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) @@ -793,6 +805,7 @@ namespace wallet_rpc std::list<std::string> tx_key_list; std::list<uint64_t> amount_list; std::list<uint64_t> fee_list; + std::list<uint64_t> weight_list; std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; @@ -803,6 +816,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_key_list) KV_SERIALIZE(amount_list) KV_SERIALIZE(fee_list) + KV_SERIALIZE(weight_list) KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) @@ -850,6 +864,7 @@ namespace wallet_rpc std::string tx_key; uint64_t amount; uint64_t fee; + uint64_t weight; std::string tx_blob; std::string tx_metadata; std::string multisig_txset; @@ -860,6 +875,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_key) KV_SERIALIZE(amount) KV_SERIALIZE(fee) + KV_SERIALIZE(weight) KV_SERIALIZE(tx_blob) KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) @@ -1591,9 +1607,13 @@ namespace wallet_rpc struct request_t { std::string data; + uint32_t account_index; + uint32_t address_index; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(data); + KV_SERIALIZE(data) + KV_SERIALIZE_OPT(account_index, 0u) + KV_SERIALIZE_OPT(address_index, 0u) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1823,12 +1843,10 @@ namespace wallet_rpc struct request_t { std::string address; - std::string payment_id; std::string description; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) - KV_SERIALIZE(payment_id) KV_SERIALIZE(description) END_KV_SERIALIZE_MAP() }; @@ -1852,8 +1870,6 @@ namespace wallet_rpc uint64_t index; bool set_address; std::string address; - bool set_payment_id; - std::string payment_id; bool set_description; std::string description; @@ -1861,8 +1877,6 @@ namespace wallet_rpc KV_SERIALIZE(index) KV_SERIALIZE(set_address) KV_SERIALIZE(address) - KV_SERIALIZE(set_payment_id) - KV_SERIALIZE(payment_id) KV_SERIALIZE(set_description) KV_SERIALIZE(description) END_KV_SERIALIZE_MAP() @@ -1893,13 +1907,11 @@ namespace wallet_rpc { uint64_t index; std::string address; - std::string payment_id; std::string description; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(index) KV_SERIALIZE(address) - KV_SERIALIZE(payment_id) KV_SERIALIZE(description) END_KV_SERIALIZE_MAP() }; @@ -2580,5 +2592,36 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_ESTIMATE_TX_SIZE_AND_WEIGHT + { + struct request_t + { + uint32_t n_inputs; + uint32_t n_outputs; + uint32_t ring_size; + bool rct; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(n_inputs) + KV_SERIALIZE(n_outputs) + KV_SERIALIZE_OPT(ring_size, 0u) + KV_SERIALIZE_OPT(rct, true) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + uint64_t size; + uint64_t weight; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(size) + KV_SERIALIZE(weight) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + } } diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index db5b7797a..9463d14ce 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -160,8 +160,8 @@ string tx2str(const cryptonote::transaction& tx, const cryptonote::hash256& tx_h return ss.str(); }*/ -bool tests::proxy_core::handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - if (!keeped_by_block) +bool tests::proxy_core::handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, cryptonote::relay_method tx_relay, bool relayed) { + if (tx_relay != cryptonote::relay_method::block) return true; crypto::hash tx_hash = null_hash; @@ -190,13 +190,13 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::tx_blob_entry& tx_b return true; } -bool tests::proxy_core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) +bool tests::proxy_core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, cryptonote::relay_method tx_relay, bool relayed) { tvc.resize(tx_blobs.size()); size_t i = 0; for (const auto &tx_blob: tx_blobs) { - if (!handle_incoming_tx(tx_blob, tvc[i], keeped_by_block, relayed, do_not_relay)) + if (!handle_incoming_tx(tx_blob, tvc[i], tx_relay, relayed)) return false; ++i; } diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 0e41a2be0..8732c85cc 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -34,6 +34,7 @@ #include "cryptonote_basic/cryptonote_basic_impl.h" #include "cryptonote_basic/verification_context.h" +#include "cryptonote_core/i_core_events.h" #include <unordered_map> namespace tests @@ -51,7 +52,7 @@ namespace tests : height(_height), id(_id), longhash(_longhash), blk(_blk), blob(_blob), txes(_txes) { } }; - class proxy_core + class proxy_core : public cryptonote::i_core_events { cryptonote::block m_genesis; std::list<crypto::hash> m_known_block_list; @@ -75,8 +76,8 @@ namespace tests bool get_stat_info(cryptonote::core_stat_info& st_inf){return true;} bool have_block(const crypto::hash& id); void get_blockchain_top(uint64_t& height, crypto::hash& top_id); - bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, cryptonote::relay_method tx_relay, bool relayed); + bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, cryptonote::relay_method tx_relay, bool relayed); bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true); void pause_mine(){} void resume_mine(){} @@ -90,9 +91,9 @@ namespace tests bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; } uint64_t get_target_blockchain_height() const { return 1; } size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } - virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {} + virtual void on_transactions_relayed(epee::span<const cryptonote::blobdata> tx_blobs, cryptonote::relay_method tx_relay) {} cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; } - bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } + bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob, cryptonote::relay_category tx_category) const { return false; } bool pool_has_tx(const crypto::hash &txid) const { return false; } bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return false; } bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return false; } diff --git a/tests/core_tests/CMakeLists.txt b/tests/core_tests/CMakeLists.txt index f93cbf3ad..42e76e613 100644 --- a/tests/core_tests/CMakeLists.txt +++ b/tests/core_tests/CMakeLists.txt @@ -40,6 +40,7 @@ set(core_tests_sources ring_signature_1.cpp transaction_tests.cpp tx_validation.cpp + tx_pool.cpp v2_tests.cpp rct.cpp bulletproofs.cpp @@ -57,6 +58,7 @@ set(core_tests_headers integer_overflow.h multisig.h ring_signature_1.h + tx_pool.h transaction_tests.h tx_validation.h v2_tests.h diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 35e449e10..f935d4f64 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -47,6 +47,12 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/miner.h" +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/testdb.h" + #include "chaingen.h" #include "device/device.hpp" using namespace std; @@ -55,6 +61,126 @@ using namespace epee; using namespace crypto; using namespace cryptonote; +namespace +{ + /** + * Dummy TestDB to store height -> (block, hash) information + * for the use only in the test_generator::fill_nonce() function, + * which requires blockchain object to correctly compute PoW on HF12+ blocks + * as the mining function requires it to obtain a valid seedhash. + */ + class TestDB: public cryptonote::BaseTestDB + { + private: + struct block_t + { + cryptonote::block bl; + crypto::hash hash; + }; + + public: + TestDB() { m_open = true; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) override + { + blocks.push_back({blk, blk_hash}); + } + + virtual uint64_t height() const override { return blocks.empty() ? 0 : blocks.size() - 1; } + + // Required for randomx + virtual crypto::hash get_block_hash_from_height(const uint64_t &height) const override + { + if (height < blocks.size()) + { + MDEBUG("Get hash for block height: " << height << " hash: " << blocks[height].hash); + return blocks[height].hash; + } + + MDEBUG("Get hash for block height: " << height << " zero-hash"); + crypto::hash hash = crypto::null_hash; + *(uint64_t*)&hash = height; + return hash; + } + + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override + { + const uint64_t h = height(); + if (block_height != nullptr) + { + *block_height = h; + } + + return get_block_hash_from_height(h); + } + + virtual cryptonote::block get_top_block() const override + { + if (blocks.empty()) + { + cryptonote::block b; + return b; + } + + return blocks[blocks.size()-1].bl; + } + + virtual void pop_block(cryptonote::block &blk, std::vector<cryptonote::transaction> &txs) override { if (!blocks.empty()) blocks.pop_back(); } + virtual void set_hard_fork_version(uint64_t height, uint8_t version) override { if (height >= hf.size()) hf.resize(height + 1); hf[height] = version; } + virtual uint8_t get_hard_fork_version(uint64_t height) const override { if (height >= hf.size()) return 255; return hf[height]; } + + private: + std::vector<block_t> blocks; + std::vector<uint8_t> hf; + }; + +} + +static std::unique_ptr<cryptonote::Blockchain> init_blockchain(const std::vector<test_event_entry> & events, cryptonote::network_type nettype) +{ + std::unique_ptr<cryptonote::Blockchain> bc; + v_hardforks_t hardforks; + cryptonote::test_options test_options_tmp{nullptr, 0}; + const cryptonote::test_options * test_options = &test_options_tmp; + if (!extract_hard_forks(events, hardforks)) + { + MDEBUG("Extracting hard-forks from blocks"); + extract_hard_forks_from_blocks(events, hardforks); + } + + hardforks.push_back(std::make_pair((uint8_t)0, (uint64_t)0)); // terminator + test_options_tmp.hard_forks = hardforks.data(); + test_options = &test_options_tmp; + + cryptonote::tx_memory_pool txpool(*bc); + bc.reset(new cryptonote::Blockchain(txpool)); + + cryptonote::Blockchain *blockchain = bc.get(); + auto bdb = new TestDB(); + + BOOST_FOREACH(const test_event_entry &ev, events) + { + if (typeid(block) != ev.type()) + { + continue; + } + + const block *blk = &boost::get<block>(ev); + auto blk_hash = get_block_hash(*blk); + bdb->add_block(*blk, 1, 1, 1, 0, 0, blk_hash); + } + + bool r = blockchain->init(bdb, nettype, true, test_options, 2, nullptr); + CHECK_AND_ASSERT_THROW_MES(r, "could not init blockchain from events"); + return bc; +} void test_generator::get_block_chain(std::vector<block_info>& blockchain, const crypto::hash& head, size_t n) const { @@ -184,13 +310,7 @@ bool test_generator::construct_block(cryptonote::block& blk, uint64_t height, co //blk.tree_root_hash = get_tx_tree_hash(blk); - // Nonce search... - blk.nonce = 0; - while (!miner::find_nonce_for_given_block([](const cryptonote::block &b, uint64_t height, unsigned int threads, crypto::hash &hash){ - return cryptonote::get_block_longhash(NULL, b, hash, height, threads); - }, blk, get_test_difficulty(hf_ver), height)) - blk.timestamp++; - + fill_nonce(blk, get_test_difficulty(hf_ver), height); add_block(blk, txs_weight, block_weights, already_generated_coins, hf_ver ? hf_ver.get() : 1); return true; @@ -268,6 +388,32 @@ bool test_generator::construct_block_manually_tx(cryptonote::block& blk, const c return construct_block_manually(blk, prev_block, miner_acc, bf_tx_hashes, 0, 0, 0, crypto::hash(), 0, transaction(), tx_hashes, txs_weight); } +void test_generator::fill_nonce(cryptonote::block& blk, const difficulty_type& diffic, uint64_t height) +{ + const cryptonote::Blockchain *blockchain = nullptr; + std::unique_ptr<cryptonote::Blockchain> bc; + + if (blk.major_version >= RX_BLOCK_VERSION && diffic > 1) + { + if (m_events == nullptr) + { + MDEBUG("events not set, RandomX PoW can fail due to zero seed hash"); + } + else + { + bc = init_blockchain(*m_events, m_nettype); + blockchain = bc.get(); + } + } + + blk.nonce = 0; + while (!miner::find_nonce_for_given_block([blockchain](const cryptonote::block &b, uint64_t height, unsigned int threads, crypto::hash &hash){ + return cryptonote::get_block_longhash(blockchain, b, hash, height, threads); + }, blk, diffic, height)) { + blk.timestamp++; + } +} + namespace { uint64_t get_inputs_amount(const vector<tx_source_entry> &s) @@ -796,15 +942,6 @@ void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& event fill_tx_sources_and_destinations(events, blk_head, from, to.get_keys().m_account_address, amount, fee, nmix, sources, destinations); } -void fill_nonce(cryptonote::block& blk, const difficulty_type& diffic, uint64_t height) -{ - blk.nonce = 0; - while (!miner::find_nonce_for_given_block([](const cryptonote::block &b, uint64_t height, unsigned int threads, crypto::hash &hash){ - return cryptonote::get_block_longhash(NULL, b, hash, height, threads); - }, blk, diffic, height)) - blk.timestamp++; -} - cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr, uint64_t amount) { tx_destination_entry de; @@ -983,6 +1120,31 @@ bool extract_hard_forks(const std::vector<test_event_entry>& events, v_hardforks return !hard_forks.empty(); } +bool extract_hard_forks_from_blocks(const std::vector<test_event_entry>& events, v_hardforks_t& hard_forks) +{ + int hf = -1; + int64_t height = 0; + + for(auto & ev : events) + { + if (typeid(block) != ev.type()) + { + continue; + } + + const block *blk = &boost::get<block>(ev); + if (blk->major_version != hf) + { + hf = blk->major_version; + hard_forks.push_back(std::make_pair(blk->major_version, (uint64_t)height)); + } + + height += 1; + } + + return !hard_forks.empty(); +} + void get_confirmed_txs(const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx, map_hash2tx_t& confirmed_txs) { std::unordered_set<crypto::hash> confirmed_hashes; diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index b78640dc9..80ce7404b 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -47,12 +47,14 @@ #include "include_base_utils.h" #include "common/boost_serialization_helper.h" #include "common/command_line.h" +#include "common/threadpool.h" #include "cryptonote_basic/account_boost_serialization.h" #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/cryptonote_basic_impl.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/enums.h" #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "misc_language.h" @@ -108,17 +110,17 @@ typedef serialized_object<cryptonote::transaction> serialized_transaction; struct event_visitor_settings { - int valid_mask; - bool txs_keeped_by_block; + int mask; enum settings { - set_txs_keeped_by_block = 1 << 0 + set_txs_keeped_by_block = 1 << 0, + set_txs_do_not_relay = 1 << 1, + set_local_relay = 1 << 2 }; - event_visitor_settings(int a_valid_mask = 0, bool a_txs_keeped_by_block = false) - : valid_mask(a_valid_mask) - , txs_keeped_by_block(a_txs_keeped_by_block) + event_visitor_settings(int a_mask = 0) + : mask(a_mask) { } @@ -128,8 +130,7 @@ private: template<class Archive> void serialize(Archive & ar, const unsigned int /*version*/) { - ar & valid_mask; - ar & txs_keeped_by_block; + ar & mask; } }; @@ -227,8 +228,8 @@ public: bf_hf_version= 1 << 8 }; - test_generator() {} - test_generator(const test_generator &other): m_blocks_info(other.m_blocks_info) {} + test_generator(): m_events(nullptr) {} + test_generator(const test_generator &other): m_blocks_info(other.m_blocks_info), m_events(other.m_events), m_nettype(other.m_nettype) {} void get_block_chain(std::vector<block_info>& blockchain, const crypto::hash& head, size_t n) const; void get_last_n_block_weights(std::vector<size_t>& block_weights, const crypto::hash& head, size_t n) const; uint64_t get_already_generated_coins(const crypto::hash& blk_id) const; @@ -253,9 +254,14 @@ public: uint8_t hf_version = 1); bool construct_block_manually_tx(cryptonote::block& blk, const cryptonote::block& prev_block, const cryptonote::account_base& miner_acc, const std::vector<crypto::hash>& tx_hashes, size_t txs_size); + void fill_nonce(cryptonote::block& blk, const cryptonote::difficulty_type& diffic, uint64_t height); + void set_events(const std::vector<test_event_entry> * events) { m_events = events; } + void set_network_type(const cryptonote::network_type nettype) { m_nettype = nettype; } private: std::unordered_map<crypto::hash, block_info> m_blocks_info; + const std::vector<test_event_entry> * m_events; + cryptonote::network_type m_nettype; friend class boost::serialization::access; @@ -407,7 +413,6 @@ cryptonote::account_public_address get_address(const cryptonote::tx_destination_ inline cryptonote::difficulty_type get_test_difficulty(const boost::optional<uint8_t>& hf_ver=boost::none) {return !hf_ver || hf_ver.get() <= 1 ? 1 : 2;} inline uint64_t current_difficulty_window(const boost::optional<uint8_t>& hf_ver=boost::none){ return !hf_ver || hf_ver.get() <= 1 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; } -void fill_nonce(cryptonote::block& blk, const cryptonote::difficulty_type& diffic, uint64_t height); cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr=false, uint64_t amount=0); std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1=false, uint64_t am1=0); @@ -490,6 +495,7 @@ void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& event uint64_t get_balance(const cryptonote::account_base& addr, const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx); bool extract_hard_forks(const std::vector<test_event_entry>& events, v_hardforks_t& hard_forks); +bool extract_hard_forks_from_blocks(const std::vector<test_event_entry>& events, v_hardforks_t& hard_forks); /************************************************************************/ /* */ @@ -503,7 +509,7 @@ private: t_test_class& m_validator; size_t m_ev_index; - bool m_txs_keeped_by_block; + cryptonote::relay_method m_tx_relay; public: push_core_event_visitor(cryptonote::core& c, const std::vector<test_event_entry>& events, t_test_class& validator) @@ -511,7 +517,7 @@ public: , m_events(events) , m_validator(validator) , m_ev_index(0) - , m_txs_keeped_by_block(false) + , m_tx_relay(cryptonote::relay_method::fluff) { } @@ -530,9 +536,21 @@ public: { log_event("event_visitor_settings"); - if (settings.valid_mask & event_visitor_settings::set_txs_keeped_by_block) + if (settings.mask & event_visitor_settings::set_txs_keeped_by_block) { - m_txs_keeped_by_block = settings.txs_keeped_by_block; + m_tx_relay = cryptonote::relay_method::block; + } + else if (settings.mask & event_visitor_settings::set_local_relay) + { + m_tx_relay = cryptonote::relay_method::local; + } + else if (settings.mask & event_visitor_settings::set_txs_do_not_relay) + { + m_tx_relay = cryptonote::relay_method::none; + } + else + { + m_tx_relay = cryptonote::relay_method::fluff; } return true; @@ -544,7 +562,7 @@ public: cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); size_t pool_size = m_c.get_pool_transactions_count(); - m_c.handle_incoming_tx({t_serializable_object_to_blob(tx), crypto::null_hash}, tvc, m_txs_keeped_by_block, false, false); + m_c.handle_incoming_tx({t_serializable_object_to_blob(tx), crypto::null_hash}, tvc, m_tx_relay, false); bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); bool r = m_validator.check_tx_verification_context(tvc, tx_added, m_ev_index, tx); CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); @@ -564,7 +582,7 @@ public: tvcs.push_back(tvc0); } size_t pool_size = m_c.get_pool_transactions_count(); - m_c.handle_incoming_txs(tx_blobs, tvcs, m_txs_keeped_by_block, false, false); + m_c.handle_incoming_txs(tx_blobs, tvcs, m_tx_relay, false); size_t tx_added = m_c.get_pool_transactions_count() - pool_size; bool r = m_validator.check_tx_verification_context_array(tvcs, tx_added, m_ev_index, txs); CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); @@ -644,7 +662,7 @@ public: cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); size_t pool_size = m_c.get_pool_transactions_count(); - m_c.handle_incoming_tx(sr_tx.data, tvc, m_txs_keeped_by_block, false, false); + m_c.handle_incoming_tx(sr_tx.data, tvc, m_tx_relay, false); bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); cryptonote::transaction tx; @@ -758,6 +776,7 @@ inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cry t_test_class validator; bool ret = replay_events_through_core<t_test_class>(c, events, validator); + tools::threadpool::getInstance().recycle(); // c.deinit(); return ret; } @@ -955,7 +974,7 @@ inline bool do_replay_file(const std::string& filename) #define MAKE_MINER_TX_MANUALLY(TX, BLK) MAKE_MINER_TX_AND_KEY_MANUALLY(TX, BLK, 0) -#define SET_EVENT_VISITOR_SETT(VEC_EVENTS, SETT, VAL) VEC_EVENTS.push_back(event_visitor_settings(SETT, VAL)); +#define SET_EVENT_VISITOR_SETT(VEC_EVENTS, SETT) VEC_EVENTS.push_back(event_visitor_settings(SETT)); #define GENERATE(filename, genclass) \ { \ diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 8406f416f..23f3170b8 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -32,6 +32,7 @@ #include "chaingen_tests_list.h" #include "common/util.h" #include "common/command_line.h" +#include "tx_pool.h" #include "transaction_tests.h" namespace po = boost::program_options; @@ -155,6 +156,12 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(gen_tx_output_is_not_txout_to_key); GENERATE_AND_PLAY(gen_tx_signatures_are_invalid); + // Mempool + GENERATE_AND_PLAY(txpool_spend_key_public); + GENERATE_AND_PLAY(txpool_spend_key_all); + GENERATE_AND_PLAY(txpool_double_spend_norelay); + GENERATE_AND_PLAY(txpool_double_spend_local); + // Double spend GENERATE_AND_PLAY(gen_double_spend_in_tx<false>); GENERATE_AND_PLAY(gen_double_spend_in_tx<true>); diff --git a/tests/core_tests/double_spend.cpp b/tests/core_tests/double_spend.cpp index afd212b27..4aa12c8e0 100644 --- a/tests/core_tests/double_spend.cpp +++ b/tests/core_tests/double_spend.cpp @@ -46,7 +46,7 @@ bool gen_double_spend_in_different_chains::generate(std::vector<test_event_entry { INIT_DOUBLE_SPEND_TEST(); - SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block, true); + SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block); MAKE_TX(events, tx_1, bob_account, alice_account, send_amount / 2 - TESTS_DEFAULT_FEE, blk_1); events.pop_back(); MAKE_TX(events, tx_2, bob_account, alice_account, send_amount - TESTS_DEFAULT_FEE, blk_1); @@ -96,3 +96,4 @@ bool gen_double_spend_in_different_chains::check_double_spend(cryptonote::core& return true; } + diff --git a/tests/core_tests/double_spend.inl b/tests/core_tests/double_spend.inl index 600899b6f..684c9c4de 100644 --- a/tests/core_tests/double_spend.inl +++ b/tests/core_tests/double_spend.inl @@ -147,7 +147,7 @@ bool gen_double_spend_in_tx<txs_keeped_by_block>::generate(std::vector<test_even if (!construct_tx(bob_account.get_keys(), sources, destinations, boost::none, std::vector<uint8_t>(), tx_1, 0)) return false; - SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block, txs_keeped_by_block); + SET_EVENT_VISITOR_SETT(events, txs_keeped_by_block ? event_visitor_settings::set_txs_keeped_by_block : 0); DO_CALLBACK(events, "mark_invalid_tx"); events.push_back(tx_1); DO_CALLBACK(events, "mark_invalid_block"); @@ -163,7 +163,7 @@ bool gen_double_spend_in_the_same_block<txs_keeped_by_block>::generate(std::vect INIT_DOUBLE_SPEND_TEST(); DO_CALLBACK(events, "mark_last_valid_block"); - SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block, txs_keeped_by_block); + SET_EVENT_VISITOR_SETT(events, txs_keeped_by_block ? event_visitor_settings::set_txs_keeped_by_block : 0); MAKE_TX_LIST_START(events, txs_1, bob_account, alice_account, send_amount - TESTS_DEFAULT_FEE, blk_1); cryptonote::transaction tx_1 = txs_1.front(); @@ -190,7 +190,7 @@ bool gen_double_spend_in_different_blocks<txs_keeped_by_block>::generate(std::ve INIT_DOUBLE_SPEND_TEST(); DO_CALLBACK(events, "mark_last_valid_block"); - SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block, txs_keeped_by_block); + SET_EVENT_VISITOR_SETT(events, txs_keeped_by_block ? event_visitor_settings::set_txs_keeped_by_block : 0); // Create two identical transactions, but don't push it to events list MAKE_TX(events, tx_blk_2, bob_account, alice_account, send_amount - TESTS_DEFAULT_FEE, blk_1); @@ -220,7 +220,7 @@ bool gen_double_spend_in_alt_chain_in_the_same_block<txs_keeped_by_block>::gener { INIT_DOUBLE_SPEND_TEST(); - SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block, txs_keeped_by_block); + SET_EVENT_VISITOR_SETT(events, txs_keeped_by_block ? event_visitor_settings::set_txs_keeped_by_block : 0); // Main chain MAKE_NEXT_BLOCK(events, blk_2, blk_1r, miner_account); @@ -255,7 +255,7 @@ bool gen_double_spend_in_alt_chain_in_different_blocks<txs_keeped_by_block>::gen { INIT_DOUBLE_SPEND_TEST(); - SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block, txs_keeped_by_block); + SET_EVENT_VISITOR_SETT(events, txs_keeped_by_block ? event_visitor_settings::set_txs_keeped_by_block : 0); // Main chain MAKE_NEXT_BLOCK(events, blk_2, blk_1r, miner_account); diff --git a/tests/core_tests/tx_pool.cpp b/tests/core_tests/tx_pool.cpp new file mode 100644 index 000000000..537015dca --- /dev/null +++ b/tests/core_tests/tx_pool.cpp @@ -0,0 +1,561 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "tx_pool.h" + +#include <boost/chrono/chrono.hpp> +#include <boost/thread/thread_only.hpp> +#include <limits> +#include "string_tools.h" + +#define INIT_MEMPOOL_TEST() \ + uint64_t send_amount = 1000; \ + uint64_t ts_start = 1338224400; \ + GENERATE_ACCOUNT(miner_account); \ + GENERATE_ACCOUNT(bob_account); \ + MAKE_GENESIS_BLOCK(events, blk_0, miner_account, ts_start); \ + REWIND_BLOCKS(events, blk_0r, blk_0, miner_account); \ + + +txpool_base::txpool_base() + : test_chain_unit_base() + , m_broadcasted_tx_count(0) + , m_all_tx_count(0) +{ + REGISTER_CALLBACK_METHOD(txpool_spend_key_public, increase_broadcasted_tx_count); + REGISTER_CALLBACK_METHOD(txpool_spend_key_public, increase_all_tx_count); + REGISTER_CALLBACK_METHOD(txpool_spend_key_public, check_txpool_spent_keys); +} + +bool txpool_base::increase_broadcasted_tx_count(cryptonote::core& /*c*/, size_t /*ev_index*/, const std::vector<test_event_entry>& /*events*/) +{ + ++m_broadcasted_tx_count; + return true; +} + +bool txpool_base::increase_all_tx_count(cryptonote::core& /*c*/, size_t /*ev_index*/, const std::vector<test_event_entry>& /*events*/) +{ + ++m_all_tx_count; + return true; +} + +bool txpool_base::check_txpool_spent_keys(cryptonote::core& c, size_t /*ev_index*/, const std::vector<test_event_entry>& events) +{ + std::vector<cryptonote::tx_info> infos{}; + std::vector<cryptonote::spent_key_image_info> key_images{}; + if (!c.get_pool_transactions_and_spent_keys_info(infos, key_images) || infos.size() != m_broadcasted_tx_count || key_images.size() != m_broadcasted_tx_count) + { + MERROR("Failed broadcasted spent keys retrieval - Expected Broadcasted Count: " << m_broadcasted_tx_count << " Actual Info Count: " << infos.size() << " Actual Key Image Count: " << key_images.size()); + return false; + } + + infos.clear(); + key_images.clear(); + if (!c.get_pool_transactions_and_spent_keys_info(infos, key_images, false) || infos.size() != m_broadcasted_tx_count || key_images.size() != m_broadcasted_tx_count) + { + MERROR("Failed broadcasted spent keys retrieval - Expected Broadcasted Count: " << m_broadcasted_tx_count << " Actual Info Count: " << infos.size() << " Actual Key Image Count: " << key_images.size()); + return false; + } + + infos.clear(); + key_images.clear(); + if (!c.get_pool_transactions_and_spent_keys_info(infos, key_images, true) || infos.size() != m_all_tx_count || key_images.size() != m_all_tx_count) + { + MERROR("Failed all spent keys retrieval - Expected All Count: " << m_all_tx_count << " Actual Info Count: " << infos.size() << " Actual Key Image Count: " << key_images.size()); + return false; + } + + return true; +} + +bool txpool_spend_key_public::generate(std::vector<test_event_entry>& events) const +{ + INIT_MEMPOOL_TEST(); + + DO_CALLBACK(events, "check_txpool_spent_keys"); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + DO_CALLBACK(events, "increase_broadcasted_tx_count"); + DO_CALLBACK(events, "increase_all_tx_count"); + DO_CALLBACK(events, "check_txpool_spent_keys"); + + return true; +} + +bool txpool_spend_key_all::generate(std::vector<test_event_entry>& events) +{ + INIT_MEMPOOL_TEST(); + SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_do_not_relay); + + DO_CALLBACK(events, "check_txpool_spent_keys"); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + DO_CALLBACK(events, "increase_all_tx_count"); + DO_CALLBACK(events, "check_txpool_spent_keys"); + + return true; +} + +txpool_double_spend_base::txpool_double_spend_base() + : txpool_base() + , m_broadcasted_hashes() + , m_no_relay_hashes() + , m_all_hashes() + , m_no_new_index(0) + , m_new_timestamp_index(0) + , m_last_tx(crypto::hash{}) +{ + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, mark_no_new); + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, mark_timestamp_change); + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, timestamp_change_pause); + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, check_unchanged); + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, check_new_broadcasted); + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, check_new_hidden); + REGISTER_CALLBACK_METHOD(txpool_double_spend_base, check_new_no_relay); +} + +bool txpool_double_spend_base::mark_no_new(cryptonote::core& /*c*/, size_t ev_index, const std::vector<test_event_entry>& /*events*/) +{ + m_no_new_index = ev_index + 1; + return true; +} + +bool txpool_double_spend_base::mark_timestamp_change(cryptonote::core& /*c*/, size_t ev_index, const std::vector<test_event_entry>& /*events*/) +{ + m_new_timestamp_index = ev_index + 1; + return true; +} + +bool txpool_double_spend_base::timestamp_change_pause(cryptonote::core& /*c*/, size_t /*ev_index*/, const std::vector<test_event_entry>& /*events*/) +{ + boost::this_thread::sleep_for(boost::chrono::seconds{1} + boost::chrono::milliseconds{100}); + return true; +} + +bool txpool_double_spend_base::check_changed(cryptonote::core& c, const size_t ev_index, relay_test condition) +{ + const std::size_t public_hash_count = m_broadcasted_hashes.size(); + const std::size_t all_hash_count = m_all_hashes.size(); + + const std::size_t new_broadcasted_hash_count = m_broadcasted_hashes.size() + unsigned(condition == relay_test::broadcasted); + const std::size_t new_all_hash_count = m_all_hashes.size() + unsigned(condition == relay_test::hidden) + unsigned(condition == relay_test::no_relay); + + std::vector<crypto::hash> hashes{}; + if (!c.get_pool_transaction_hashes(hashes)) + { + MERROR("Failed to get broadcasted transaction pool hashes"); + return false; + } + + for (const crypto::hash& hash : hashes) + m_broadcasted_hashes.insert(hash); + + if (new_broadcasted_hash_count != m_broadcasted_hashes.size()) + { + MERROR("Expected " << new_broadcasted_hash_count << " broadcasted hashes but got " << m_broadcasted_hashes.size()); + return false; + } + + if (m_broadcasted_hashes.size() != c.get_pool_transactions_count()) + { + MERROR("Expected " << m_broadcasted_hashes.size() << " broadcasted hashes but got " << c.get_pool_transactions_count()); + return false; + } + + hashes.clear(); + if (!c.get_pool_transaction_hashes(hashes, false)) + { + MERROR("Failed to get broadcasted transaction pool hashes"); + return false; + } + + for (const crypto::hash& hash : hashes) + m_all_hashes.insert(std::make_pair(hash, 0)); + + if (new_broadcasted_hash_count != m_broadcasted_hashes.size()) + { + MERROR("Expected " << new_broadcasted_hash_count << " broadcasted hashes but got " << m_broadcasted_hashes.size()); + return false; + } + + hashes.clear(); + if (!c.get_pool_transaction_hashes(hashes, true)) + { + + MERROR("Failed to get all transaction pool hashes"); + return false; + } + + for (const crypto::hash& hash : hashes) + m_all_hashes.insert(std::make_pair(hash, 0)); + + if (new_all_hash_count != m_all_hashes.size()) + { + MERROR("Expected " << new_all_hash_count << " all hashes but got " << m_all_hashes.size()); + return false; + } + + if (condition == relay_test::no_relay) + { + if (!m_no_relay_hashes.insert(m_last_tx).second) + { + MERROR("Expected new no_relay tx but got a duplicate legacy tx"); + return false; + } + + for (const crypto::hash& hash : m_no_relay_hashes) + { + if (!c.pool_has_tx(hash)) + { + MERROR("Expected public tx " << hash << " to be listed in pool"); + return false; + } + } + } + + // check receive time changes + { + std::vector<cryptonote::tx_info> infos{}; + std::vector<cryptonote::spent_key_image_info> key_images{}; + if (!c.get_pool_transactions_and_spent_keys_info(infos, key_images, true) || infos.size() != m_all_hashes.size()) + { + MERROR("Unable to retrieve all txpool metadata"); + return false; + } + + for (const cryptonote::tx_info& info : infos) + { + crypto::hash tx_hash; + if (!epee::string_tools::hex_to_pod(info.id_hash, tx_hash)) + { + MERROR("Unable to convert tx_hash hex to binary"); + return false; + } + + const auto entry = m_all_hashes.find(tx_hash); + if (entry == m_all_hashes.end()) + { + MERROR("Unable to find tx_hash in set of tracked hashes"); + return false; + } + + if (m_new_timestamp_index == ev_index && m_last_tx == tx_hash) + { + if (entry->second >= info.receive_time) + { + MERROR("Last relay time did not change as expected - last at " << entry->second << " and current at " << info.receive_time); + return false; + } + entry->second = info.receive_time; + } + else if (entry->second != info.receive_time) + { + MERROR("Last relayed time changed unexpectedly from " << entry->second << " to " << info.receive_time); + return false; + } + } + } + + { + std::vector<cryptonote::transaction> txes{}; + if (!c.get_pool_transactions(txes)) + { + MERROR("Failed to get broadcasted transactions from pool"); + return false; + } + + hashes.clear(); + for (const cryptonote::transaction& tx : txes) + hashes.push_back(cryptonote::get_transaction_hash(tx)); + + std::unordered_set<crypto::hash> public_hashes = m_broadcasted_hashes; + for (const crypto::hash& hash : hashes) + { + if (!c.pool_has_tx(hash)) + { + MERROR("Expected broadcasted tx " << hash << " to be listed in pool"); + return false; + } + + if (!public_hashes.erase(hash)) + { + MERROR("An unexected transaction was returned from the public pool"); + return false; + } + } + if (!public_hashes.empty()) + { + MERROR(public_hashes.size() << " transaction(s) were missing from the public pool"); + return false; + } + } + + { + std::vector<cryptonote::transaction> txes{}; + if (!c.get_pool_transactions(txes, false)) + { + MERROR("Failed to get broadcasted transactions from pool"); + return false; + } + + hashes.clear(); + for (const cryptonote::transaction& tx : txes) + hashes.push_back(cryptonote::get_transaction_hash(tx)); + + std::unordered_set<crypto::hash> public_hashes = m_broadcasted_hashes; + for (const crypto::hash& hash : hashes) + { + + if (!public_hashes.erase(hash)) + { + MERROR("An unexected transaction was returned from the public pool"); + return false; + } + } + if (!public_hashes.empty()) + { + MERROR(public_hashes.size() << " transaction(s) were missing from the public pool"); + return false; + } + } + + { + std::vector<cryptonote::transaction> txes{}; + if (!c.get_pool_transactions(txes, true)) + { + MERROR("Failed to get all transactions from pool"); + return false; + } + + hashes.clear(); + for (const cryptonote::transaction& tx : txes) + hashes.push_back(cryptonote::get_transaction_hash(tx)); + + std::unordered_map<crypto::hash, uint64_t> all_hashes = m_all_hashes; + for (const crypto::hash& hash : hashes) + { + if (!all_hashes.erase(hash)) + { + MERROR("An unexected transaction was returned from the all pool"); + return false; + } + } + if (!all_hashes.empty()) + { + MERROR(m_broadcasted_hashes.size() << " transaction(s) were missing from the all pool"); + return false; + } + } + + { + std::vector<cryptonote::tx_backlog_entry> entries{}; + if (!c.get_txpool_backlog(entries)) + { + MERROR("Failed to get broadcasted txpool backlog"); + return false; + } + + if (m_broadcasted_hashes.size() != entries.size()) + { + MERROR("Expected " << m_broadcasted_hashes.size() << " in the broadcasted txpool backlog but got " << entries.size()); + return false; + } + } + + for (const std::pair<crypto::hash, uint64_t>& hash : m_all_hashes) + { + cryptonote::blobdata tx_blob{}; + if (!c.get_pool_transaction(hash.first, tx_blob, cryptonote::relay_category::all)) + { + MERROR("Failed to retrieve tx expected to be in pool: " << hash.first); + return false; + } + } + + { + std::unordered_map<crypto::hash, uint64_t> difference = m_all_hashes; + for (const crypto::hash& hash : m_broadcasted_hashes) + difference.erase(hash); + + for (const crypto::hash& hash : m_no_relay_hashes) + difference.erase(hash); + + for (const std::pair<crypto::hash, uint64_t>& hash : difference) + { + if (c.pool_has_tx(hash.first)) + { + MERROR("Did not expect private/hidden tx " << hash.first << " to be listed in pool"); + return false; + } + + cryptonote::blobdata tx_blob{}; + if (c.get_pool_transaction(hash.first, tx_blob, cryptonote::relay_category::broadcasted)) + { + MERROR("Tx " << hash.first << " is not supposed to be in broadcasted pool"); + return false; + } + + if (!c.get_pool_transaction(hash.first, tx_blob, cryptonote::relay_category::all)) + { + MERROR("Tx " << hash.first << " blob could not be retrieved from pool"); + return false; + } + } + } + + { + cryptonote::txpool_stats stats{}; + if (!c.get_pool_transaction_stats(stats) || stats.txs_total != m_broadcasted_hashes.size()) + { + MERROR("Expected broadcasted stats to list " << m_broadcasted_hashes.size() << " txes but got " << stats.txs_total); + return false; + } + + if (!c.get_pool_transaction_stats(stats, false) || stats.txs_total != m_broadcasted_hashes.size()) + { + MERROR("Expected broadcasted stats to list " << m_broadcasted_hashes.size() << " txes but got " << stats.txs_total); + return false; + } + + if (!c.get_pool_transaction_stats(stats, true) || stats.txs_total != m_all_hashes.size()) + { + MERROR("Expected all stats to list " << m_all_hashes.size() << " txes but got " << stats.txs_total); + return false; + } + } + + { + std::vector<cryptonote::rpc::tx_in_pool> infos{}; + cryptonote::rpc::key_images_with_tx_hashes key_images{}; + if (!c.get_pool_for_rpc(infos, key_images) || infos.size() != m_broadcasted_hashes.size() || key_images.size() != m_broadcasted_hashes.size()) + { + MERROR("Expected broadcasted rpc data to return " << m_broadcasted_hashes.size() << " but got " << infos.size() << " infos and " << key_images.size() << "key images"); + return false; + } + } + return true; +} + +bool txpool_double_spend_base::check_unchanged(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& /*events */) +{ + return check_changed(c, ev_index, relay_test::no_change); +} + +bool txpool_double_spend_base::check_new_broadcasted(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& /*events */) +{ + return check_changed(c, ev_index, relay_test::broadcasted); +} + +bool txpool_double_spend_base::check_new_hidden(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& /*events */) +{ + return check_changed(c, ev_index, relay_test::hidden); +} +bool txpool_double_spend_base::check_new_no_relay(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& /*events */) +{ + return check_changed(c, ev_index, relay_test::no_relay); +} + +bool txpool_double_spend_base::check_tx_verification_context(const cryptonote::tx_verification_context& tvc, bool tx_added, size_t event_idx, const cryptonote::transaction& tx) +{ + m_last_tx = cryptonote::get_transaction_hash(tx); + if (m_no_new_index == event_idx) + return !tvc.m_verifivation_failed && !tx_added; + else + return !tvc.m_verifivation_failed && tx_added; +} + +bool txpool_double_spend_norelay::generate(std::vector<test_event_entry>& events) const +{ + INIT_MEMPOOL_TEST(); + + DO_CALLBACK(events, "check_txpool_spent_keys"); + SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_do_not_relay); + DO_CALLBACK(events, "mark_no_new"); + + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + + DO_CALLBACK(events, "increase_all_tx_count"); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "mark_timestamp_change"); + DO_CALLBACK(events, "check_new_no_relay"); + DO_CALLBACK(events, "timestamp_change_pause"); + DO_CALLBACK(events, "mark_no_new"); + events.push_back(tx_0); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "check_unchanged"); + SET_EVENT_VISITOR_SETT(events, 0); + DO_CALLBACK(events, "timestamp_change_pause"); + DO_CALLBACK(events, "mark_no_new"); + events.push_back(tx_0); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "check_unchanged"); + + // kepped by block currently does not change txpool status + SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_keeped_by_block); + DO_CALLBACK(events, "timestamp_change_pause"); + DO_CALLBACK(events, "mark_no_new"); + events.push_back(tx_0); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "check_unchanged"); + + return true; +} + +bool txpool_double_spend_local::generate(std::vector<test_event_entry>& events) const +{ + INIT_MEMPOOL_TEST(); + + DO_CALLBACK(events, "check_txpool_spent_keys"); + SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_local_relay); + DO_CALLBACK(events, "mark_no_new"); + + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + + DO_CALLBACK(events, "increase_all_tx_count"); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "mark_timestamp_change"); + DO_CALLBACK(events, "check_new_hidden"); + DO_CALLBACK(events, "timestamp_change_pause"); + DO_CALLBACK(events, "mark_no_new"); + events.push_back(tx_0); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "mark_timestamp_change"); + DO_CALLBACK(events, "check_unchanged"); + SET_EVENT_VISITOR_SETT(events, 0); + DO_CALLBACK(events, "timestamp_change_pause"); + events.push_back(tx_0); + DO_CALLBACK(events, "increase_broadcasted_tx_count"); + DO_CALLBACK(events, "check_txpool_spent_keys"); + DO_CALLBACK(events, "mark_timestamp_change"); + DO_CALLBACK(events, "check_new_broadcasted"); + DO_CALLBACK(events, "timestamp_change_pause"); + DO_CALLBACK(events, "mark_no_new"); + events.push_back(tx_0); + DO_CALLBACK(events, "check_unchanged"); + + return true; +} + diff --git a/tests/core_tests/tx_pool.h b/tests/core_tests/tx_pool.h new file mode 100644 index 000000000..996c76698 --- /dev/null +++ b/tests/core_tests/tx_pool.h @@ -0,0 +1,118 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <unordered_map> +#include <unordered_set> + +#include "chaingen.h" +#include "crypto/crypto.h" + +enum class relay_test +{ + no_change = 0, //!< No expected changes to the txpool + broadcasted, //!< A new block or fluff/flood tx is expected in txpool + hidden, //!< A new stem or local tx is expected in txpool + no_relay //!< A new no relay is expected in txpool +}; + +class txpool_base : public test_chain_unit_base +{ + size_t m_broadcasted_tx_count; + size_t m_all_tx_count; + +public: + txpool_base(); + + bool increase_broadcasted_tx_count(cryptonote::core& c, size_t /*ev_index*/, const std::vector<test_event_entry>& events); + bool increase_all_tx_count(cryptonote::core& c, size_t /*ev_index*/, const std::vector<test_event_entry>& events); + bool check_txpool_spent_keys(cryptonote::core& c, size_t /*ev_index*/, const std::vector<test_event_entry>& events); +}; + +struct txpool_spend_key_public : txpool_base +{ + txpool_spend_key_public() : txpool_base() + {} + + bool generate(std::vector<test_event_entry>& events) const; +}; + +struct txpool_spend_key_all : txpool_base +{ + txpool_spend_key_all() : txpool_base() + {} + + bool generate(std::vector<test_event_entry>& events); +}; + +class txpool_double_spend_base : public txpool_base +{ + std::unordered_set<crypto::hash> m_broadcasted_hashes; + std::unordered_set<crypto::hash> m_no_relay_hashes; + std::unordered_map<crypto::hash, uint64_t> m_all_hashes; + size_t m_no_new_index; + size_t m_new_timestamp_index; + crypto::hash m_last_tx; + + bool check_changed(cryptonote::core& c, size_t ev_index, relay_test condition); + +public: + txpool_double_spend_base(); + + bool mark_no_new(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + bool mark_timestamp_change(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + + //! Pause for 1 second, so that `receive_time` for tx meta changes (tx hidden from public rpc being updated) + bool timestamp_change_pause(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + + bool check_unchanged(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + bool check_new_broadcasted(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + bool check_new_hidden(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + bool check_new_no_relay(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); + + bool check_tx_verification_context(const cryptonote::tx_verification_context& tvc, bool tx_added, size_t event_idx, const cryptonote::transaction& /*tx*/); +}; + +struct txpool_double_spend_norelay : txpool_double_spend_base +{ + txpool_double_spend_norelay() + : txpool_double_spend_base() + {} + + bool generate(std::vector<test_event_entry>& events) const; +}; + +struct txpool_double_spend_local : txpool_double_spend_base +{ + txpool_double_spend_local() + : txpool_double_spend_base() + {} + + bool generate(std::vector<test_event_entry>& events) const; +}; diff --git a/tests/core_tests/wallet_tools.cpp b/tests/core_tests/wallet_tools.cpp index 21a9455c0..fdc4753f9 100644 --- a/tests/core_tests/wallet_tools.cpp +++ b/tests/core_tests/wallet_tools.cpp @@ -10,9 +10,6 @@ using namespace epee; using namespace crypto; using namespace cryptonote; -// Shared random generator -static std::default_random_engine RND(crypto::rand<unsigned>()); - void wallet_accessor_test::set_account(tools::wallet2 * wallet, cryptonote::account_base& account) { wallet->clear(); diff --git a/tests/functional_tests/address_book.py b/tests/functional_tests/address_book.py index 8d8711ffc..f9ec217af 100755 --- a/tests/functional_tests/address_book.py +++ b/tests/functional_tests/address_book.py @@ -73,14 +73,13 @@ class AddressBookTest(): # add one res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', description = 'self') - assert res.index == 0 + assert res.index == 0, res for get_all in [True, False]: res = wallet.get_address_book() if get_all else wallet.get_address_book([0]) assert len(res.entries) == 1 e = res.entries[0] assert e.index == 0 - assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' - assert e.payment_id == '' or e.payment_id == '0' * 16 or e.payment_id == '0' * 64 + assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', e assert e.description == 'self' # add a duplicate @@ -91,7 +90,6 @@ class AddressBookTest(): assert res.entries[0].index == 0 assert res.entries[1].index == 1 assert res.entries[0].address == res.entries[1].address - assert res.entries[0].payment_id == res.entries[1].payment_id assert res.entries[0].description == res.entries[1].description e = res.entries[1] res = wallet.get_address_book([1]) @@ -118,7 +116,6 @@ class AddressBookTest(): assert len(res.entries) == 1 assert res.entries[0].index == 0 assert res.entries[0].address == e.address - assert res.entries[0].payment_id == e.payment_id assert res.entries[0].description == e.description # delete (new) first @@ -165,38 +162,13 @@ class AddressBookTest(): assert res.entries[0] == e assert res.entries[1] == e - # payment IDs - res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 64) - assert res.index == 2 - ok = False - try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = 'x' * 64) - except: ok = True - assert ok - ok = False - try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 65) - except: ok = True - assert ok - ok = False - try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 63) - except: ok = True - assert ok - ok = False - try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 16) - except: ok = True - assert ok - # various address types res = wallet.make_integrated_address() integrated_address = res.integrated_address - integrated_address_payment_id = res.payment_id - ok = False - try: res = wallet.add_address_book(integrated_address, payment_id = '0' * 64) - except: ok = True - assert ok res = wallet.add_address_book(integrated_address) - assert res.index == 3 + assert res.index == 2 res = wallet.add_address_book('87KfgTZ8ER5D3Frefqnrqif11TjVsTPaTcp37kqqKMrdDRUhpJRczeR7KiBmSHF32UJLP3HHhKUDmEQyJrv2mV8yFDCq8eB') - assert res.index == 4 + assert res.index == 3 # get them back res = wallet.get_address_book([0]) @@ -209,16 +181,9 @@ class AddressBookTest(): assert res.entries[0].description == u'あまやかす' res = wallet.get_address_book([2]) assert len(res.entries) == 1 - assert res.entries[0].address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.entries[0].address == integrated_address res = wallet.get_address_book([3]) assert len(res.entries) == 1 - if False: # for now, the address book splits integrated addresses - assert res.entries[0].address == integrated_address - else: - assert res.entries[0].address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' - assert res.entries[0].payment_id == integrated_address_payment_id + '0' * 48 - res = wallet.get_address_book([4]) - assert len(res.entries) == 1 assert res.entries[0].address == '87KfgTZ8ER5D3Frefqnrqif11TjVsTPaTcp37kqqKMrdDRUhpJRczeR7KiBmSHF32UJLP3HHhKUDmEQyJrv2mV8yFDCq8eB' # edit @@ -227,15 +192,12 @@ class AddressBookTest(): e = res.entries[0] assert e.index == 1 assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' - assert e.payment_id == '0' * 64 assert e.description == u'あまやかす' - res = wallet.edit_address_book(1, payment_id = '1' * 64) res = wallet.get_address_book([1]) assert len(res.entries) == 1 e = res.entries[0] assert e.index == 1 assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' - assert e.payment_id == '1' * 64 assert e.description == u'あまやかす' res = wallet.edit_address_book(1, description = '') res = wallet.get_address_book([1]) @@ -243,7 +205,6 @@ class AddressBookTest(): e = res.entries[0] assert e.index == 1 assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' - assert e.payment_id == '1' * 64 assert e.description == '' res = wallet.edit_address_book(1, description = 'えんしゅう') res = wallet.get_address_book([1]) @@ -251,7 +212,6 @@ class AddressBookTest(): e = res.entries[0] assert e.index == 1 assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' - assert e.payment_id == '1' * 64 assert e.description == u'えんしゅう' res = wallet.edit_address_book(1, address = '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A') res = wallet.get_address_book([1]) @@ -259,25 +219,12 @@ class AddressBookTest(): e = res.entries[0] assert e.index == 1 assert e.address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' - assert e.payment_id == '1' * 64 - assert e.description == u'えんしゅう' - res = wallet.edit_address_book(1, payment_id = '') - res = wallet.get_address_book([1]) - assert len(res.entries) == 1 - e = res.entries[0] - assert e.index == 1 - assert e.address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' - assert e.payment_id == '0' * 64 assert e.description == u'えんしゅう' ok = False try: res = wallet.edit_address_book(1, address = '') except: ok = True assert ok ok = False - try: res = wallet.edit_address_book(1, payment_id = 'asdnd') - except: ok = True - assert ok - ok = False try: res = wallet.edit_address_book(1, address = 'address') except: ok = True assert ok @@ -287,7 +234,6 @@ class AddressBookTest(): assert e == res.entries[0] # empty - wallet.delete_address_book(4) wallet.delete_address_book(0) res = wallet.get_address_book([0]) # entries above the deleted one collapse one slot up assert len(res.entries) == 1 diff --git a/tests/functional_tests/make_test_signature.cc b/tests/functional_tests/make_test_signature.cc index 8c0333233..789523de5 100644 --- a/tests/functional_tests/make_test_signature.cc +++ b/tests/functional_tests/make_test_signature.cc @@ -32,6 +32,7 @@ int main(int argc, const char **argv) { + TRY_ENTRY(); if (argc > 2) { fprintf(stderr, "usage: %s <secret_key>\n", argv[0]); @@ -57,4 +58,5 @@ int main(int argc, const char **argv) std::string signature = cryptonote::make_rpc_payment_signature(skey); printf("%s\n", signature.c_str()); return 0; + CATCH_ENTRY_L0("main()", 1); } diff --git a/tests/functional_tests/mining.py b/tests/functional_tests/mining.py index ad646417e..d067c25e1 100755 --- a/tests/functional_tests/mining.py +++ b/tests/functional_tests/mining.py @@ -170,5 +170,15 @@ class MiningTest(): assert res.hash == block_hash +class Guard: + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + daemon = Daemon() + try: daemon.stop_mining() + except: pass + if __name__ == '__main__': - MiningTest().run_test() + with Guard() as guard: + MiningTest().run_test() diff --git a/tests/functional_tests/sign_message.py b/tests/functional_tests/sign_message.py index de8f0cee2..9dd70f8bc 100755 --- a/tests/functional_tests/sign_message.py +++ b/tests/functional_tests/sign_message.py @@ -43,7 +43,8 @@ from framework.wallet import Wallet class MessageSigningTest(): def run_test(self): self.create() - self.check_signing() + self.check_signing(False) + self.check_signing(True) def create(self): print('Creating wallets') @@ -65,20 +66,34 @@ class MessageSigningTest(): assert res.address == self.address[i] assert res.seed == seeds[i] - def check_signing(self): - print('Signing/verifing messages') + def check_signing(self, subaddress): + print('Signing/verifing messages with ' + ('subaddress' if subaddress else 'standard address')) messages = ['foo', ''] + if subaddress: + address = [] + for i in range(2): + res = self.wallet[i].create_account() + if i == 0: + account_index = res.account_index + res = self.wallet[i].create_address(account_index = account_index) + if i == 0: + address_index = res.address_index + address.append(res.address) + else: + address = [self.address[0], self.address[1]] + account_index = 0 + address_index = 0 for message in messages: - res = self.wallet[0].sign(message) + res = self.wallet[0].sign(message, account_index = account_index, address_index = address_index) signature = res.signature for i in range(2): - res = self.wallet[i].verify(message, self.address[0], signature) + res = self.wallet[i].verify(message, address[0], signature) assert res.good - res = self.wallet[i].verify('different', self.address[0], signature) + res = self.wallet[i].verify('different', address[0], signature) assert not res.good - res = self.wallet[i].verify(message, self.address[1], signature) + res = self.wallet[i].verify(message, address[1], signature) assert not res.good - res = self.wallet[i].verify(message, self.address[0], signature + 'x') + res = self.wallet[i].verify(message, address[0], signature + 'x') assert not res.good if __name__ == '__main__': diff --git a/tests/net_load_tests/clt.cpp b/tests/net_load_tests/clt.cpp index fc2280f23..e154363e7 100644 --- a/tests/net_load_tests/clt.cpp +++ b/tests/net_load_tests/clt.cpp @@ -202,11 +202,11 @@ namespace // Connect to server std::atomic<int> conn_status(0); - m_cmd_conn_id = boost::uuids::nil_uuid(); + m_context = {}; ASSERT_TRUE(m_tcp_server.connect_async("127.0.0.1", srv_port, CONNECTION_TIMEOUT, [&](const test_connection_context& context, const boost::system::error_code& ec) { if (!ec) { - m_cmd_conn_id = context.m_connection_id; + m_context = context; } else { @@ -217,11 +217,11 @@ namespace EXPECT_TRUE(busy_wait_for(DEFAULT_OPERATION_TIMEOUT, [&]{ return 0 != conn_status.load(std::memory_order_seq_cst); })) << "connect_async timed out"; ASSERT_EQ(1, conn_status.load(std::memory_order_seq_cst)); - ASSERT_FALSE(m_cmd_conn_id.is_nil()); + ASSERT_FALSE(m_context.m_connection_id.is_nil()); conn_status.store(0, std::memory_order_seq_cst); CMD_RESET_STATISTICS::request req; - ASSERT_TRUE(epee::net_utils::async_invoke_remote_command2<CMD_RESET_STATISTICS::response>(m_cmd_conn_id, CMD_RESET_STATISTICS::ID, req, + ASSERT_TRUE(epee::net_utils::async_invoke_remote_command2<CMD_RESET_STATISTICS::response>(m_context, CMD_RESET_STATISTICS::ID, req, m_tcp_server.get_config_object(), [&](int code, const CMD_RESET_STATISTICS::response& rsp, const test_connection_context&) { conn_status.store(code, std::memory_order_seq_cst); })); @@ -250,16 +250,16 @@ namespace // Connect to server and invoke shutdown command std::atomic<int> conn_status(0); - boost::uuids::uuid cmd_conn_id = boost::uuids::nil_uuid(); + test_connection_context cmd_context; tcp_server.connect_async("127.0.0.1", srv_port, CONNECTION_TIMEOUT, [&](const test_connection_context& context, const boost::system::error_code& ec) { - cmd_conn_id = context.m_connection_id; + cmd_context = context; conn_status.store(!ec ? 1 : -1, std::memory_order_seq_cst); }); if (!busy_wait_for(DEFAULT_OPERATION_TIMEOUT, [&]{ return 0 != conn_status.load(std::memory_order_seq_cst); })) return; if (1 != conn_status.load(std::memory_order_seq_cst)) return; - epee::net_utils::notify_remote_command2(cmd_conn_id, CMD_SHUTDOWN::ID, CMD_SHUTDOWN::request(), tcp_server.get_config_object()); + epee::net_utils::notify_remote_command2(cmd_context, CMD_SHUTDOWN::ID, CMD_SHUTDOWN::request(), tcp_server.get_config_object()); busy_wait_for(DEFAULT_OPERATION_TIMEOUT, [&]{ return 0 != commands_handler.close_connection_counter(); }); } @@ -299,7 +299,7 @@ namespace { std::atomic<int> req_status(0); CMD_GET_STATISTICS::request req; - ASSERT_TRUE(epee::net_utils::async_invoke_remote_command2<CMD_GET_STATISTICS::response>(m_cmd_conn_id, CMD_GET_STATISTICS::ID, req, + ASSERT_TRUE(epee::net_utils::async_invoke_remote_command2<CMD_GET_STATISTICS::response>(m_context, CMD_GET_STATISTICS::ID, req, m_tcp_server.get_config_object(), [&](int code, const CMD_GET_STATISTICS::response& rsp, const test_connection_context&) { if (0 < code) { @@ -338,14 +338,14 @@ namespace { CMD_SEND_DATA_REQUESTS::request req; req.request_size = request_size; - epee::net_utils::notify_remote_command2(m_cmd_conn_id, CMD_SEND_DATA_REQUESTS::ID, req, m_tcp_server.get_config_object()); + epee::net_utils::notify_remote_command2(m_context, CMD_SEND_DATA_REQUESTS::ID, req, m_tcp_server.get_config_object()); } protected: test_tcp_server m_tcp_server; test_levin_commands_handler m_commands_handler; size_t m_thread_count; - boost::uuids::uuid m_cmd_conn_id; + test_connection_context m_context; }; } @@ -434,7 +434,7 @@ TEST_F(net_load_test_clt, a_lot_of_client_connections_and_connections_closed_by_ // Close connections CMD_CLOSE_ALL_CONNECTIONS::request req; - ASSERT_TRUE(epee::net_utils::notify_remote_command2(m_cmd_conn_id, CMD_CLOSE_ALL_CONNECTIONS::ID, req, m_tcp_server.get_config_object())); + ASSERT_TRUE(epee::net_utils::notify_remote_command2(m_context, CMD_CLOSE_ALL_CONNECTIONS::ID, req, m_tcp_server.get_config_object())); // Wait for all opened connections to close busy_wait_for(DEFAULT_OPERATION_TIMEOUT, [&](){ return m_commands_handler.new_connection_counter() - RESERVED_CONN_CNT <= m_commands_handler.close_connection_counter(); }); @@ -455,10 +455,10 @@ TEST_F(net_load_test_clt, a_lot_of_client_connections_and_connections_closed_by_ // Close rest connections m_tcp_server.get_config_object().foreach_connection([&](test_connection_context& ctx) { - if (ctx.m_connection_id != m_cmd_conn_id) + if (ctx.m_connection_id != m_context.m_connection_id) { CMD_DATA_REQUEST::request req; - bool r = epee::net_utils::async_invoke_remote_command2<CMD_DATA_REQUEST::response>(ctx.m_connection_id, CMD_DATA_REQUEST::ID, req, + bool r = epee::net_utils::async_invoke_remote_command2<CMD_DATA_REQUEST::response>(ctx, CMD_DATA_REQUEST::ID, req, m_tcp_server.get_config_object(), [=](int code, const CMD_DATA_REQUEST::response& rsp, const test_connection_context&) { if (code <= 0) { @@ -548,7 +548,7 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser CMD_START_OPEN_CLOSE_TEST::request req_start; req_start.open_request_target = CONNECTION_COUNT; req_start.max_opened_conn_count = MAX_OPENED_CONN_COUNT; - ASSERT_TRUE(epee::net_utils::async_invoke_remote_command2<CMD_START_OPEN_CLOSE_TEST::response>(m_cmd_conn_id, CMD_START_OPEN_CLOSE_TEST::ID, req_start, + ASSERT_TRUE(epee::net_utils::async_invoke_remote_command2<CMD_START_OPEN_CLOSE_TEST::response>(m_context, CMD_START_OPEN_CLOSE_TEST::ID, req_start, m_tcp_server.get_config_object(), [&](int code, const CMD_START_OPEN_CLOSE_TEST::response&, const test_connection_context&) { test_state.store(0 < code ? 1 : -1, std::memory_order_seq_cst); })); @@ -582,7 +582,7 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser // Ask server to close rest connections CMD_CLOSE_ALL_CONNECTIONS::request req; - ASSERT_TRUE(epee::net_utils::notify_remote_command2(m_cmd_conn_id, CMD_CLOSE_ALL_CONNECTIONS::ID, req, m_tcp_server.get_config_object())); + ASSERT_TRUE(epee::net_utils::notify_remote_command2(m_context, CMD_CLOSE_ALL_CONNECTIONS::ID, req, m_tcp_server.get_config_object())); // Wait for almost all connections to be closed by server busy_wait_for(DEFAULT_OPERATION_TIMEOUT, [&](){ return m_commands_handler.new_connection_counter() <= m_commands_handler.close_connection_counter() + RESERVED_CONN_CNT; }); @@ -601,10 +601,10 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser // Close rest connections m_tcp_server.get_config_object().foreach_connection([&](test_connection_context& ctx) { - if (ctx.m_connection_id != m_cmd_conn_id) + if (ctx.m_connection_id != m_context.m_connection_id) { CMD_DATA_REQUEST::request req; - bool r = epee::net_utils::async_invoke_remote_command2<CMD_DATA_REQUEST::response>(ctx.m_connection_id, CMD_DATA_REQUEST::ID, req, + bool r = epee::net_utils::async_invoke_remote_command2<CMD_DATA_REQUEST::response>(ctx, CMD_DATA_REQUEST::ID, req, m_tcp_server.get_config_object(), [=](int code, const CMD_DATA_REQUEST::response& rsp, const test_connection_context&) { if (code <= 0) { diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index cdc2d267a..4a76f2ec6 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -47,6 +47,7 @@ namespace net_load_tests { struct test_connection_context : epee::net_utils::connection_context_base { + test_connection_context(): epee::net_utils::connection_context_base(boost::uuids::nil_uuid(), {}, false, false), m_closed(false) {} volatile bool m_closed; }; diff --git a/tests/net_load_tests/srv.cpp b/tests/net_load_tests/srv.cpp index fe32ec5cb..b42b1e1b0 100644 --- a/tests/net_load_tests/srv.cpp +++ b/tests/net_load_tests/srv.cpp @@ -147,7 +147,7 @@ namespace CMD_DATA_REQUEST::request req2; req2.data.resize(req.request_size); - bool r = epee::net_utils::async_invoke_remote_command2<CMD_DATA_REQUEST::response>(ctx.m_connection_id, CMD_DATA_REQUEST::ID, req2, + bool r = epee::net_utils::async_invoke_remote_command2<CMD_DATA_REQUEST::response>(ctx, CMD_DATA_REQUEST::ID, req2, m_tcp_server.get_config_object(), [=](int code, const CMD_DATA_REQUEST::response& rsp, const test_connection_context&) { if (code <= 0) { diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 96825f54f..cda25dfc9 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -92,7 +92,8 @@ set(unit_tests_sources ringdb.cpp wipeable_string.cpp is_hdd.cpp - aligned.cpp) + aligned.cpp + rpc_version_str.cpp) set(unit_tests_headers unit_tests_utils.h) diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index e5ca4e41e..38707f075 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -271,12 +271,12 @@ namespace EXPECT_EQ(connection_ids_.size(), connections_->get_connections_count()); } - cryptonote::levin::notify make_notifier(const std::size_t noise_size, bool is_public) + cryptonote::levin::notify make_notifier(const std::size_t noise_size, bool is_public, bool pad_txs) { epee::byte_slice noise = nullptr; if (noise_size) noise = epee::levin::make_noise_notify(noise_size); - return cryptonote::levin::notify{io_service_, connections_, std::move(noise), is_public}; + return cryptonote::levin::notify{io_service_, connections_, std::move(noise), is_public, pad_txs}; } boost::uuids::random_generator random_generator_; @@ -434,12 +434,16 @@ TEST_F(levin_notify, defaulted) EXPECT_FALSE(status.has_noise); EXPECT_FALSE(status.connections_filled); } - EXPECT_FALSE(notifier.send_txs({}, random_generator_(), false)); + EXPECT_TRUE(notifier.send_txs({}, random_generator_())); + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + EXPECT_FALSE(notifier.send_txs(std::move(txs), random_generator_())); } -TEST_F(levin_notify, flood) +TEST_F(levin_notify, fluff_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true); + cryptonote::levin::notify notifier = make_notifier(0, true, false); for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -464,10 +468,13 @@ TEST_F(levin_notify, flood) ASSERT_EQ(10u, contexts_.size()); { auto context = contexts_.begin(); - EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), false)); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id())); io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); + notifier.run_fluff(); + ASSERT_LT(0u, io_service_.poll()); + EXPECT_EQ(0u, context->process_send_queue()); for (++context; context != contexts_.end(); ++context) EXPECT_EQ(1u, context->process_send_queue()); @@ -480,14 +487,42 @@ TEST_F(levin_notify, flood) EXPECT_TRUE(notification._.empty()); } } +} + +TEST_F(levin_notify, fluff_with_padding) +{ + cryptonote::levin::notify notifier = make_notifier(0, true, true); + + for (unsigned count = 0; count < 10; ++count) + add_connection(count % 2 == 0); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); // not tracked + } + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + txs[1].resize(200, 'f'); ASSERT_EQ(10u, contexts_.size()); { auto context = contexts_.begin(); - EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), true)); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id())); io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); + notifier.run_fluff(); + ASSERT_LT(0u, io_service_.poll()); + EXPECT_EQ(0u, context->process_send_queue()); for (++context; context != contexts_.end(); ++context) EXPECT_EQ(1u, context->process_send_queue()); @@ -502,9 +537,9 @@ TEST_F(levin_notify, flood) } } -TEST_F(levin_notify, private_flood) +TEST_F(levin_notify, private_fluff_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false); + cryptonote::levin::notify notifier = make_notifier(0, false, false); for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -529,10 +564,14 @@ TEST_F(levin_notify, private_flood) ASSERT_EQ(10u, contexts_.size()); { auto context = contexts_.begin(); - EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), false)); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id())); io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); + notifier.run_fluff(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + EXPECT_EQ(0u, context->process_send_queue()); for (++context; context != contexts_.end(); ++context) { @@ -548,14 +587,43 @@ TEST_F(levin_notify, private_flood) EXPECT_TRUE(notification._.empty()); } } +} + +TEST_F(levin_notify, private_fluff_with_padding) +{ + cryptonote::levin::notify notifier = make_notifier(0, false, true); + + for (unsigned count = 0; count < 10; ++count) + add_connection(count % 2 == 0); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); // not tracked + } + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + txs[1].resize(200, 'f'); ASSERT_EQ(10u, contexts_.size()); { auto context = contexts_.begin(); - EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), true)); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id())); io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); + notifier.run_fluff(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + EXPECT_EQ(0u, context->process_send_queue()); for (++context; context != contexts_.end(); ++context) { @@ -582,7 +650,7 @@ TEST_F(levin_notify, noise) txs[0].resize(1900, 'h'); const boost::uuids::uuid incoming_id = random_generator_(); - cryptonote::levin::notify notifier = make_notifier(2048, false); + cryptonote::levin::notify notifier = make_notifier(2048, false, true); { const auto status = notifier.get_status(); @@ -608,7 +676,7 @@ TEST_F(levin_notify, noise) EXPECT_EQ(0u, receiver_.notified_size()); } - EXPECT_TRUE(notifier.send_txs(txs, incoming_id, false)); + EXPECT_TRUE(notifier.send_txs(txs, incoming_id)); notifier.run_stems(); io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); @@ -627,7 +695,7 @@ TEST_F(levin_notify, noise) } txs[0].resize(3000, 'r'); - EXPECT_TRUE(notifier.send_txs(txs, incoming_id, true)); + EXPECT_TRUE(notifier.send_txs(txs, incoming_id)); notifier.run_stems(); io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index 5f91fc6d4..c92f70b97 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -32,6 +32,7 @@ #include "cryptonote_core/cryptonote_core.h" #include "p2p/net_node.h" #include "p2p/net_node.inl" +#include "cryptonote_core/i_core_events.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "cryptonote_protocol/cryptonote_protocol_handler.inl" @@ -43,7 +44,7 @@ namespace cryptonote { class blockchain_storage; } -class test_core +class test_core : public cryptonote::i_core_events { public: void on_synchronized(){} @@ -56,8 +57,8 @@ public: bool get_stat_info(cryptonote::core_stat_info& st_inf) const {return true;} bool have_block(const crypto::hash& id) const {return true;} void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;} - bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } - bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } + bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, cryptonote::relay_method tx_relay, bool relayed) { return true; } + bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, cryptonote::relay_method tx_relay, bool relayed) { return true; } bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; } void pause_mine(){} void resume_mine(){} @@ -71,9 +72,9 @@ public: bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; } uint64_t get_target_blockchain_height() const { return 1; } size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } - virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {} + virtual void on_transactions_relayed(epee::span<const cryptonote::blobdata> tx_blobs, cryptonote::relay_method tx_relay) {} cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; } - bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } + bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob, cryptonote::relay_category tx_category) const { return false; } bool pool_has_tx(const crypto::hash &txid) const { return false; } bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return false; } bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return false; } diff --git a/tests/unit_tests/rpc_version_str.cpp b/tests/unit_tests/rpc_version_str.cpp new file mode 100644 index 000000000..5dce60465 --- /dev/null +++ b/tests/unit_tests/rpc_version_str.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "gtest/gtest.h" + +#include "rpc/rpc_version_str.h" +#include "version.h" + +TEST(rpc, is_version_string_valid) +{ + using namespace cryptonote::rpc; + ASSERT_TRUE(is_version_string_valid(MONERO_VERSION)); + ASSERT_TRUE(is_version_string_valid("0.14.1.2")); + ASSERT_TRUE(is_version_string_valid("0.15.0.0-release")); + ASSERT_TRUE(is_version_string_valid("0.15.0.0-fe3f6a3e6")); + + ASSERT_FALSE(is_version_string_valid("")); + ASSERT_FALSE(is_version_string_valid("invalid")); + ASSERT_FALSE(is_version_string_valid("0.15.0.0-invalid")); + ASSERT_FALSE(is_version_string_valid("0.15.0.0-release0")); + ASSERT_FALSE(is_version_string_valid("0.15.0.0-release ")); + ASSERT_FALSE(is_version_string_valid("0.15.0.0-fe3f6a3e60")); + ASSERT_FALSE(is_version_string_valid("0.15.0.0-fe3f6a3e6 ")); +} diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 23f028464..b711526e6 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -735,7 +735,6 @@ TEST(Serialization, portability_wallet) auto address_book_row = w.m_address_book.begin(); ASSERT_TRUE(epee::string_tools::pod_to_hex(address_book_row->m_address.m_spend_public_key) == "9bc53a6ff7b0831c9470f71b6b972dbe5ad1e8606f72682868b1dda64e119fb3"); ASSERT_TRUE(epee::string_tools::pod_to_hex(address_book_row->m_address.m_view_public_key) == "49fece1ef97dc0c0f7a5e2106e75e96edd910f7e86b56e1e308cd0cf734df191"); - ASSERT_TRUE(epee::string_tools::pod_to_hex(address_book_row->m_payment_id) == "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); ASSERT_TRUE(address_book_row->m_description == "testnet wallet 9y52S6"); } } diff --git a/utils/gpg_keys/binaryfate.asc b/utils/gpg_keys/binaryfate.asc new file mode 100644 index 000000000..55c5b161c --- /dev/null +++ b/utils/gpg_keys/binaryfate.asc @@ -0,0 +1,87 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF3yDrwBEAC1UgCSLILsbdrSk5kfcEYKMvj7lJpIIj9D6LeeyIvovgO7beM0 +63cFCT0v+RH3CVKV8bCCJr8teR3Zgk+IeI6C0CQk+ocqlu0qBAALdZyGyZonozbc +lHGOfQ0rWEy01V/TB36bGhrsE/cM8nhICJ72Pkv3rrukZkprxvEJ+IYCk26Umiue +K1+Pm0sUMrxAQUYlvg8r1swOgLOuo7r8c1gZYvGixdb0t7mBzUgkSdmFUeAa/X8W +WzBPFluWMyetGUKzrV66W1ISHHi/2AyXim235Lqc4MbK2ObKfkZJCjC7y2afs5MS +t+uejz8bchLMM/LvV2TxKIbenho7ZxtGd8blNRAPe6FTOA+yOM50atJah1W3BmLx +sxk4gvII2/zlzaW4RNEy1Ma/47DINPwYB9BA7FqF7BTVt8WxpJ5Y2+8aR58ZtM+q +ObbO5s2O8kuDj94qQKAT8btetbb/pMKhF3XXSARkNZPzNFFtSy9xSin+hLAWpM0b +cftEtJrE7HY7DHOa9J4P/xFqLSQnZGpClg5lyRw34kU1l8sAzovFngN2Zn+RYkiW +y14I+uhVOFzroH2ymIYTwQ0crQJ4OYgtqzv0Rc+U2mTrTE+MRG28COsBqoMvWao9 +K4bhsP6qUmlTtMVFob4AP3eFqF3wg987zqeeCqaidAc7n5fuJHTDGNWFxQARAQAB +tCViaW5hcnlGYXRlIDxiaW5hcnlmYXRlQGdldG1vbmVyby5vcmc+iQJOBBMBCgA4 +FiEEgaxZH+nEtlxYBq/D8K9NRioL35IFAl3yDrwCGy8FCwkIBwIGFQoJCAsCBBYC +AwECHgECF4AACgkQ8K9NRioL35IT2w/+JaVcSVo3vXTpC9TOSxnyjFP1QuPVxn+K +P4la6MzTQlaZOSaoGuXra+82ghWqJGbFhGYmsXWQZtUYVwTMq9N9IUJ55clfkiV3 +HQNIDh/6zz+ibBzxfNLAPTbz0OQ+26dWRmBKkjpIs8KWyVIR/Ma/ax1No20fPKoN +cgRvOct/BKNoX2+PlHh/0tTTIjc0NqNP79ptNyJUUTGL1uZOgfTGa7/FRxil1xiC +JZfbZ2mbdesqjTtF3CI/ahhPKUEKy5Tciy5pnErALPk4rsxdLNdl3NOVrb5tYqGv +GqoQT3KsHkYVHvLCx0Ji2peFa6lF0PyJpudUzlo7L3nn39b8v9ebKd/fk2gthnWr +eoFxEFa6dI0x+HA47FE00I7ze56oH0zFVUhx+3QOHFG3u2mHV+H+GfKaXq9Nd5nU +Wk8AGKJ/xYgX7G0TnUek9aOzSt3dRd+xtSjmgwA3T7vCi6F67SthRpq7F4zuecPl ++Ww1MQhXSdIi6P3ll09gTCOICaqOxuOkMeFFrs2w4XEU517QUzy1VTAcF3gSy9TL +Xpy5jqssjLjvDAHs/Jm+iryQtn/t9YHr7/DP7AAr4+e7aOY3659jP7UiMZ3RKqY5 +/15Ru/usXXFyC56Epsv1spBCthimiK/RdX8XW7mG+BIQJXN337WdicXGWrKVOixj +eJ7ghiSblIeJAjMEEAEKAB0WIQTsEs+GtgjVvRuRCZGpI7HrTHj9WAUCXfOqVwAK +CRCpI7HrTHj9WPSaD/wMa2PWbRj06DUHiKRMrrQF9gkzox2ZOd4om4YIa3opA+GJ +CRl6nMHO7MdGTiun0/jBoT4oU4rqvPYcQlKOXUFYaMz/3vPYyTotC8n1Rvf+DhQZ +CIubdrRW0B5pqa3MiyJ9S/oMf/5otto3eBZSnfegI0J/fNzyDAyS2WIj/iyoAXGQ +yy9lY+KQnjchIHItxFm6RWgZmbO6xbhRq+LUa90jEiSul1PoE+ldpH2wUlRvjP78 +lL4YqWC7U6puhgGPlQV2gvFG4zscXVf8XXV65KysqrB5RI8wx1JxC1n1ocOqwYbw +0P6WIUGSr98cBjzB1l9zKhHumV7AM9FNUf4GD+Mgl8I8i/vOSz8kYRaWCcoRZvWa +pgSEG4uA94MFZUP9b789quoMUv8kTboHU1XFDo/n7K83AVBThcj0BMt7ueZmwjtu +2YVc3vGX7GvsUr8XXYn82hqnHcBoVLI4awGMzzAiOI1KSPtksGr36mYgOuY1jYme +435YlVxH4A/NC1qvOZYLq/6u1pB/BouBtlVxCvyjhlM3/r0PgDB0KveiR72LzVXH +TB0CzmyizZnVYbNR1PvsxFvyPxRLIIfN1nzXFYxwILASlg0zhdkQgFWkZ+WJWA6g +6HrAmrrA8iV7x6ITR6dXrT2fSLdQ+nYTntaXunUw74gGWYaMPe/GTvgcguFoU4kC +MwQQAQoAHRYhBOyzccBByW6YhVJsYUZMOzgUXzoUBQJd8647AAoJEEZMOzgUXzoU +fvgP/jI1w6DxC4qRlEwsIpjvyKQbCWYr9w817gKRPOI0ZHbjoL2zBnBwAWvgEKXV +QTOkldwpWVHvt5qZweO89bJxQFShFdQklEbVXNlf9hMZqbWHt6gyS/65caEUT/nB +pugg6Ug3MIeszin7S2sKeKj3BT2ynJioMeNNUPntogUfWlOuhkVr3JezHV3TtxKf +yrs49AAmoAHK4DH34H7d8HuVjjr0v7gQc2Yu3MeWdR4drGmyU3uNY44fGqU9pP99 +HNW2Ec//HFIz+qgBZ63ps6qyWGOjXyrpn6k9L1lg8zilC1sJxRtYTwp1a/57Kpyu +NkR1265o2zu/cvb5oQxjCWmQPiZWKsoguHxiA93g14QdmnugHPM5L6XCqDl1fkQo +dnCy5uqlqs9Ucz6O4C1GV4Av3YsMepS2Bw5uZFyujqPUdw82oWWOwZfO2xw7kKzN +zd8OnQAKj3piMh1vh375HE+HwdN74txKDcD/HqoTMwuaqX145csUvORoABA/wU4w +yBJiOhJeqvUaWAsG0q6XQMIDYjGQ0rrtY6Ba9E/1UP3D2CpjH5fZChV1fxhCaQOi +nMSCwgA3HRiT4CnTYScsbnk7RcdOlc2Zff+cxyZfMvqcGM4R50NDYu8YnVjnyajn +No5WhvYYPQIoYaDM8b+RHGw8Zw4WpQqAqli0VoNJAux19os9uQINBF3yDrwBEADI +LMleZnQ87iFofqyMm6wd+146dPC0xINz3+ExkCcFJgwiHL4o1sF8LtXjBXVuoc33 +vK7mdU1Fm3N9D4W5tkxEW3NdHICc7r3IHqThFv6lOdckKg1t9HKWzEjWkB4/4Epr +sj7Xc1owuQkOVtkaWiXzv8e/pYM4j+21V71+8b6fuInvA1nNufawzN4m8RDDgbe9 +I9SdndJoO0vnhUrv55APVG6KJBRjT+SGrxkEa3cckhpGUsQkW8KEx1VH0Wc4NBdD +xSRzW+ZUDyMzoUD/b5kxNCqinaM3P95jmNJgqB9l/3ZEPZ0G/hqDg6EhQMe8HrYT +F90Q+QqsX35RbBrdXxjbod+GZPSTg0PExm/hBrt2tTE/t/yovn22PZeJm6Mg9trN +x08b/LLZ1L040s8WqQQUK0btG065PIVRpyIlsUEPt7W+Qdj+eJdZAjIDr04qBxUQ +bwHhnxCYH4PNQq75y/w684ZWoVW3UJE1P8AcXKx/0gp4jjrZCJTAkzfUBEc6vy6H +hzYLpAar52ALBmFGPpkV4PJDdlFX4uijsjVVf23Hi3AQnGWkUw34bftJuHYlluGg +aGXVzISn8MxqcaLUiapxMMg3pZ40m5gCHG7/Gm74xKZCQiNrxCLHn7/rgNV17U6m +aWCNGcaiCXfkbtqUPJrzK6ulDAhg3700Ok5RlVOtAwARAQABiQRsBBgBCgAgFiEE +gaxZH+nEtlxYBq/D8K9NRioL35IFAl3yDrwCGy4CQAkQ8K9NRioL35LBdCAEGQEK +AB0WIQStVkzajxZlrOeLXf0lk4OOq7H2VQUCXfIOvAAKCRAlk4OOq7H2VfMbD/oC +NxmlGhAJ2okXzcpFURNyIXTCt/Dlys+lwH2mwCjcniUcA5KsT1mV1Sb4SxsBsX18 +Fhol2wUUI/B3ELz15rOYTJq7u8TPpPnGsHtALgNwN1o90EYMRpMl7jiPJLKSWVA1 +spp7FBginaP/jkQwtcy+sHiJvmEiY2kmeB27aAA5bM8bafjzpqQ6Io/iguRVYex1 +ypH5D1Dq5bkrk8Fd+3bs2dcuB9jYy0lQlaYl8bJ4I4ZOgdO4nTKq0U0RTfYAR+wH +vpfzKrYQD7ZnX6mllXyvC+L4CujngND3GiavoOPX9JHJURrYC1WiMyKPb8nk1CrF +cIfLxFYqy535suen35T23G8B0ZY/i48ccRjI7ZrickIBSjJZmJLYevsejz+T1vqf +348WCAYisi44NL6vwAoBkvYLRw45YhzAZDZoKCv+ke7zjZAO3lnh+rR3j+QCe9gH +22gqJbhGpU54EpYYVx3GxIfaqQeHa5H02rSW8AUGyGjgdTo7x5NCAJjgtYyR08k1 +19aghaA8yi1wsbmiuz9viH2xJ1vPTgLLQE5qv0ed0uOGEeDtRZSfaWeYwk+qhier +bY3R3dzVOyan6W8uWaVgE8Ch9LttYAKa9qLoLDQE9tQLU6t1gklFfQiaLBzyPIgT +Ct6Aezwlcp2cDhxYazXjyxRTYtfshVPTXJMBF+0o+W78D/9c3f9Yv61Z1MktrHIt +5REIj5YGT3bAeDnUoGE+k+QzNx8856Hu5j4prQGav8F8+V69iqwVciK0sBoFUgI5 +JJizfVbEfmbZgl5c4S8OhApNv0Gbhj9U/XftM2Pz/m/QfsB5Az1jQ8jQGHULqEWB +d4Z+Cnt0LVmGxJkz0prlaUxrGp8NyO3A1RxWJbP6P9fPIJFFwDSer2xeq79dZGFQ +CLFxA3d8kjr9nioGZFaGK/LfbVcZBBM6VI0ziKRnWPeS3Zi50oy7+/gk7HgUvUmT +8uenHLE/kmfHg1lAycQ/TDY7/sP1Wtnbr72HEFOKDIvFIF8zaWwpaYSWrj6NRTJE +b9wa2Y5TervcvOcgzhleJP3PfDrahlfgUFtD/919cuQNG416tedumAaAlLpdBEUH +9F9FQ2Hzp8Pxm3N6LzbtgDoc6NXY2C08NPI+IwHysNIXzaH3jJ53TV8pCNvvd4Ok +Sk2fOVG/fDuZibwthAFjR3NIVJIP19xyxkalbeuLT+IUSzgN2ndojtK9eh7awMii +S/AkjI5G33OKYP0WZx+6o0i21rWXIepWgm74wMa9t9NCnNY50NIvesnG1AaXNKJ5 +D2o1uLXbwis6Fm2sMNutCkQFYsk/IFWC/Y8DsJzwkOty8gl3Q0NQi64IXpFGzUuG +lmE0kDaYQqoBKXNXcybElBID8Q== +=tkJZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/snipa.asc b/utils/gpg_keys/snipa.asc new file mode 100644 index 000000000..caee18fda --- /dev/null +++ b/utils/gpg_keys/snipa.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF369W4BEADIAbqXsSEtakMgUkLQsXTyMlZhOmt4FLoGhMEY0Tl9VjbEXr65 +Rwg6YXeJrFlxV7lAQEKxNMCcZyJM7qBeEKx9cF2Lkj/BZ6qyOmqa/gpFwjRt8sK9 +td/2QMdP9P5QN/XJOhDFpRZocq/1RBPLl5sVvHv8Oh3ZAmaFM9GFogNEOSeVbB0Q +UUIkbG2qHG1aXroIPd+yk8cj65Lhfw6nqvyevR2E26fsrzZcKJU0P7O/5rHLeKQt +4Ydis6TBvGffzIg0HnTWEJwEfJTrgEgDFMDrtAdGtyQ6uNl6qupzrg9WcOlB0f9n +3+VYsBT9NriBGW2WKeKlM7JMrsxoFHQ/1i4MZL+V5fWjOR+6kjAA8A6MBc9s9j+d +1leRyawyl4JlrcIVi5c8GtzP3b7o2oFds0aGTBeYGulohxDkf/AUJkna64ESCtsJ +QwvyRr4tanuKmydjCp0GrV9sH8Vf7WoBavoYMvcB3yc3UMzKjX08ZS3Kt8ZTWb7S +XxJg9Ve1iZXcAf7GZpN0SpIygRl5HKKP4c60QtCapp+/URpi4dmNU9y57bckhBb+ +77/6ifWEyQDBuvmhYeEsmt42cxGCfMfQs77uMTXUi+L4ZeGX1O3aozarmo+/6Cdw +BXzaQFdIMT9H6tbQnr92IbR1+6g3sw9f5kD3EvbvTRFuEiDjuxW5pXtEHwARAQAB +tDRBbGV4YW5kZXIgQmxhaXIgKFNuaXBhIC8gU25pcGEyMikgPHNuaXBhQGphZ3Rl +Y2guaW8+iQJOBBMBCgA4FiEESHJ3qL0KIJwWtwDzxkVS2HfDJHkFAl369W4CGwMF +CwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQxkVS2HfDJHnk2A/+NF9GKgP5ihlo +SY/N5KjEtaX7agZucHkQNhxYaieZdCh8Ho+iRSc0k+VSL9LHOzt+23lR85HABRE7 +bR7hnoVzLCoYVlPiyTctFL2keVDZfWv3HJbdlEd51mGMd8oiupjf6/WMIYyiJUpc +3KtWg8OiK8suRwYkjBVEo5Cf4djTlJdpNvNwnVj1xMvcN0/wSmmy9r8fmso1k18a +w2WVDFx/xacGBcUlDYpqtglAQQQaz85bjNnvm0nes/ZuKH9lstxPGpB+6UaKkrYZ +9zHwbuLPRMxqJkB/TtBXOKNyFHWXz0cfABicXy7dRmvoJDDkzE/3/Nk/ZFAHNg0G +PKvO0LIt4lVDdvKJyLhod22SffmL89+Y7H+VtxegKrG49roweflv3TStXUifoFxh +CvgUXi1XKcNXevPbLBV6Ii/PVhETqFCWO81B6LkDYAg/Eoj1HwcG1NURyEGeOSRn +gGIqhL/jY0C7okIdvhQAvtzElTHAwHpNjCiQEnLv0u8IiA74KcWKQ8o/FgPvrsr5 +WU9weoj4/E5/c9HqvP/jwKNdVkqLLQQ7GP3fHG34zKjbF+b2yg8NG1kgVruA7G7V ++jxtmMGpJDtdg22jzFE8/YIZnGuNU9kplQ4xJyQZG5Wnaitfmqw0cv101Nwtr9i2 +1FmsM9xUS3FAKyOhIzcNLH1lEljE3ym5Ag0EXfr1bgEQALvZh76QVJDL+6ttF2md +mkxyG6MZ52tfPVSLVi0nooT6EBrrf41+ga7+HKuykO5PBip21F8BZa6YdJOIPdrz +AkDbSAxmiSiD3NtWT5hokzKTr9/exeS78njdVNXBRSRohQRhLiPPtx8YJfHHNAxi +/GbymvlRCXB7PatRJdDw6FKeH0HmrxRN98Wjnnj9uV7IFnhii06+9HVa16Uai/xc +aK+0eS5GUOrY6QVssKHvE2e8BwGpv5//WZwlQkfHo6rIXAIu1Uj40jqu+Z6anG28 +5VPJn92hDArzPcBPaxZteARAgpp1osNDv7wLXXofZeWSrx7yBfUecTcJwATXmPl/ +EPRNU43XSF01uI8z2DPmzixO6JrpLg+XTTG6swU9kWQh/Ewqs8W5tuKbJWzwU9Hc +IIBl1BFWohlejx7ZcbASg3nvaI33G4WQVQMiZmofgNc4ceGxfbywYZpuw3/DWQpO +XuUyas6/MonI9H6wOWfWKbEU8rjixwdcO1wGVM1hM0/QM7UvvsO/2ogOs04ZeR0l +nffbiJPnC+AZieShIOWgB3/ru3GIer1FKPrqlgfFIx0JuraOP0Bj0tPlcD8uFnn8 +qLdxPlgpMfqmzORFpoY/yy8+D4qGYIuePysc9q2L4gfqkQGrzwRrtOYv/J5LV1t6 +HqQpd5mEe9k3rFliGaMIDz1XABEBAAGJAjYEGAEKACAWIQRIcneovQognBa3APPG +RVLYd8MkeQUCXfr1bgIbDAAKCRDGRVLYd8Mkeeq9EACplJbws2jlbswpCbkkhjUs +QKyfqZFXvcod6j7meO/abaaljbOGEvgilPISIVP3dzHrFoOzd/yIj2DvkGtAiv8E +aw0Qsw3H7ypxPutE4MSKry3HJoYdZeF1r9v4JQPyGPLvdDluhtmbN9fbJKMd1PDU +srl16qTr33gGTtx6JYodzCzAkswXnFAZawPL4Mfa46CjcrlbQskjx+rlT39YKDqN +nSdhqllqoAuOxNcYNNgmUYj1hw8C0qyRxBeKHmh5wHK+4mVwOhbKCUtaRinwnJ3Q +dRtxK91MRDNxqZ5IL+eKLoh9iOm8HvDOGkGYM9+VOB7hJ7L0bXkVuzb6bLXaSMus +fv5O++d9XJeSBNf1Tb2wcqFHMTF9VdEC4V14YM5hYCLQE2QvCUvmY4+7BVSoF62e +5AcwBAXFHiG9adYnCRsfcuIt3J/HOC7xLmZMQJQPjPnsHG5OEwq41bU/82NU1uTx +Y956SeZfGoUV+9NE59484944fOF5KV2F7VfMgGipN6w3scsP4NqD3s5S+16Fhonl +9n7fWBUfJ2Huoyef1sf20IBySCh6u2a0gibMTNM8wgcqDBfYI4XmCSmDLck/0/ff +HO3oERNyGQ1eVIZFPXnV/pTJnFpHrgtkulsCYPzcwKMoHcp59H66M9ROQV9q2yIq +nDKy8l5Mnmigd0+k1Mx+Lw== +=ACfD +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/thecharlatan.asc b/utils/gpg_keys/thecharlatan.asc new file mode 100644 index 000000000..a10a1fb54 --- /dev/null +++ b/utils/gpg_keys/thecharlatan.asc @@ -0,0 +1,316 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFnWpK8BEAClQ0TefMcY46NjErgrKRwUU84SNscBZNBV/TJh5J53vyeYS3zh +B0BedWYrX6/GlK+qm+bHh/yCtQE6SpgHUZj1FxNfM7ap9GeVhVPdEZ1uLwwwxniG +xxXVoMoNsnXFyJLS9I79nE45sWiP7O1d0R1HChVu5z7+6Evii7eeayep8i+yI5lE ++FSJkwQCTdBNa1XwqhebiVh2uBEXTrtaKmvx2qgdVzKzdtCMFNZy4Esvy5d1Q/ix +KCGBwGowH+kdT7uOpOzYHPx4phRARVvgX4gitgkj1cHBD5/EUC2vuXceqI+Y3CkM +fiu5T8QkDIfGKQxe8QdoPuVo2qUlg8U87SvvMCueAK2xfc1qZz+yaSaZbbXyAaHC +GM3gA2B24K6tCs7PSU5X2Qzd8zdcDZnEqVwY64hx9ixKDwmNQ6nHTT/zzkL+jcp6 +s8OrOuFn4EbypnipcDACHw2eFbnBX65z6DzmCGqrF29T7/1ZLw7wbW6FgM0XWSsX +fzi/Ui18IaqyU+F47e1jtvcLo2sKz8lHpHtt9K9qDvsO/ta+jFyvuLHazlqbFoAc +1FNZilvtcARY/EpjPjl5KYvO4jrVcK5wshTyrcjz8v4RtpGDpYK/zcG9UbuqMcKA +cjZc2hOdidCk2oNHZqFoxO6XJwyNksYs6sKKWknmZbP4hSjTFtByNGiiSQARAQAB +tDJTZWJhc3RpYW4gS3VuZyAoVGhlQ2hhcmxhdGFuKSA8c2ViLmt1bmdAZ21haWwu +Y29tPokBHAQQAQgABgUCWki8wAAKCRBoPXk431UVzr94B/9V1MFsThcXxEhnlkED +qLFLRanVcovfjaJgwOhZNqQHTyX7Odua+yEzqDPcXL1KKGTJJc5MTICNk5P8Hyo2 +M3HCqzaJ6uGjv044DxglOgsM9UfV2kI/1STLPrriGG4pxnnwdObXp0sylmUqV9Fc +36Zxhvf9poU42OTHyDvfVA/xmLkm6vFTRsm3pd6xFa7bAK6N8VHut6v8IM4mq7jG +vThT/EdI6+lpx9v/yVm9hcdlODTpsdN781aTSpLCJNEgVcepbFTJcmygoBDBB7CE +z0ShOSprGxn2Fp7Pem9sXfCjs7uDmScnO90++twKe73GWAfd29o2o5mK4Z/VkT83 +Wz0CiQEzBBABCAAdFiEEvaa9cEK3IcRnqXWddFXF48DNzrkFAlriLssACgkQdFXF +48DNzrkcvwf9Etbq0lQ79MLk06OPliUH9v7qIcvOtOaADt9Qoia/grSvqGuPxl7t +9KFWMrS584zOxrrkLfXpPtwrHYOllHtXSOh68HrIpXCrhJQ5PxCIhvFd8HseYPZp +g6Ta/u6S9eTaVnKOzjo7u7vqrpAVcCfsh/zflKXcm3ep3KH4zjMHDbjmWiZyRMRz +YagmK8OCiFRW0L8E0Llb/Yi+sJKQ1yem1fG7oNCS51Wa/g0EIh4eRfNmWuAyeHAL +nsAulAPW78NTIS+ZFgZvxv86Jj97DKTExMU2dGZoOlOFN0bOIt0+gcCpbTAHOc9q +KRoxc6n03SatOFyzHkbQHI5l0NPLEDXKcIkBOQQTAQgAIxYhBDfsfXsKIXzbS04A +fn+rEUJn5PoEBQJaQ9OmBYMJZgGAAAoJEH+rEUJn5PoE8vQIAIxbJzC1xFY82mm3 +tdePMhC2uwDiqwPTaSIMOlViC++JXY90ApAnNfv5Cq0TrQ3UbEhpcfmIHnNtgHpz +AxIxl1ahNyfQJuTrskhfMmg2lS2nbJi/7pq3DBbVMG/qAvdgcUql/AhgIgK29VwS +2jF6OqUAt0qPn5E1jF/KlOIZcuKO5wijvzZ/k/GlKlafWD3GgOLX/AD/d80la18U +8c0m7H4Ey469c15mH9sryiyJjUWIOL/Gs5ej9JPwv7fgEj9Gg+L23xc6Nl2cReTE +cYnn3s/c8efMp6eufZ2L1JlxSZNnU8ysGcdU+LEH0Siv20JTWX2iBtVK6o46SCEx +X3xiNAiJAhwEEAEIAAYFAlpYrx4ACgkQ+JYZaaM50unQoQ//b3ts5UNAl/rh210g +wBygxtMOdWqUBe51cO1FS9t9dmhdK08VB/yNpbk2pUnzkqVaqJ0lOQXNHg4GSq5o +9uhlgiKtq/EwamnNfrD1g5XRnQlbq0KCxDQ5ezC1EKtsXlp2SrPpwG4WqhBkhuk3 +ykf+ghKpYJtX7ZndjQPTuNoIer1rRrutTtS8z06KCklJARzjCAmxwVgTvQsNAU7j +q1uX6h91b9jykXxf5M5uv7o1eDrM4Dy++/pB5rU/4D/+l6LTP1/fheJHyTPFSGWW +p1kqqTjfCJeaFF0B7T1+USVjd+kedkAQ4DPy5ViNDfLPraV/oEFVhDjs0SayyHvy +KMcghFVqcd9Yutttfqsx7NsJal6LYYOjg8kTlW7dsuRgt0uq4tXpwleXfozHq5si +8RTsM2hRrVS5DnG/qN3apbpVQJxBNADbxiwzJ364abVbvCwux/tyJ+lj+uV1RomA +4X3cxsWb3awBhSgmFL+Jo6T0kNrCxcxCaGt9hvy1NVhoVfBy6jPuCyjHuBQnpGXk +L1rzfQxgPQVeXbTuxDgM+ExjjpD99Nv9cemYsMIhkdj3hBSTBWZ1ou+W53mvEbPb +3TVWrsiboEpfodZxq+BjQSx8E+QL8AlT9uS4u6k5uKZvv5VE/uteUWuLQofirgO9 +ylfW+44cU+DtAV8hXZ9JjChie92JAjMEEAEIAB0WIQRF3ACu/d9dXLmI7IYtpFDz +r7BGxwUCWkZ7FQAKCRAtpFDzr7BGx8xLEADgjscf0mYbUgF23Dno8IARC4mZp1Uj +7Q8uoiFqdcDeYHJ8BRcIBuJX6B9TZfC8BV1WPKzmgZhWMfR16IjRhDeF0QpClpac +vSTiAl6NVxJ4pZykyOVjuZc26io20qiNYr1cFs4AEOvY5YnRAKBDTiaUjCB9LQPI +PfJiEmjqAACVw0dIUdi+Iz3MifSRAI3WSDeSVWHFAwZ+GJYDf8QuR34wSfqsoiC9 +oIq2oSHT0PiB/oqCbpMD/vxfDkZxtEYB9lxVlUizliyHxaea4j96oEgD91sK/NCt +hCxcb65tOWIR3GpH0gyY6QIjC3XGfVLFYdG+WYQgu6cgObkrFZikpCximR1zQkdI +KOQg6OU/8UHFpqNG+AGU+V7G9d2q4BtY75oACZmaJsFYJ+gHnPcxjiOzmbxxmqAM +3FIHbiLqkRol5cjlYE16bYab03NMd0o3bAwpdxIxLONm363K1yKzFVDmtj+8eK/U +Ds+HJAFQMATccg7OUwCX3bt6IJY8iO7K+97Rh/VIb7xdJ6EjGwwkp8VjVNHsmEfs +qO34djHbVhr7UFvq8wNOdjr9G5tUZY/Kg4++lcsH1ILtAAxVbpJmGGL6aWPmIh9t +lo7gVnZFkhdotToMFVskZRXYT27ZsH32CoOwGR0ljHGMVTwJ7PzEYQblehiAc57s +tzmtN/HhEI44PokCMwQQAQgAHRYhBHG1qApj/hKw102rv+SogzZKr24WBQJaR5y4 +AAoJEOSogzZKr24WuYEP/1E+3x4q7ca+aJSenkzot/QUohs/wof6KIi/x8Mqypxo +LOnTXPzhykQelNM4pV/K//dSzcEt2h6euO36XzIZXXXsF4bhOdqq1w6Xup0dkXm7 +BX96+MVx0ZV+R+1qwQjaofHZb9IpSMgl+v/zwhySj/vK+v/D7O9oG9+6Twv571mH +RKYV0Fsr0gltO+0k2DDXB2PzY0YfJLjplbjEuvK8Y1mNbM06PqS6hMtCb4gdWMyt +4w/7sJAOq0TdnEqyq/IsM3ZpshJqiaONkWifCaHSOafe7x9hsbxeuJOcyMFjGDIw +QpDtUkbuzbfCHoOIEwRXW6SgrWw2DsbZ4PKExDo2B+tW3/G2oi3b0irsb+QfEroK +7ol6wGZf9Tbwa8olBq7f1ID5pD7XTFddC+3RlgMwWNStTeQT17QsP+M2e9ju7KsW +IfmIrM46hNrhnNHpn9hEiM5Lu6I53B2pi1nMmtMTwiNRBeRmdri05LLaomiHHh6v +MqMbucmTQKYbVw8VvvgOT+LmKPYIL+VEFQb+WjnQnDjm7ewUXABA2KnNapWcf3c5 +gI5DPbUl6A0yZTAuqO+ICxj36uZC7LEfO9AazrGvtXDdUP7xOs2iqrPafiyymanC +hoBG8mPhUnyZQtdFZRxxQD9lxVbLt8uFRaQ/PQaBkmIHDGv86pGniTl2VNzBATIT +iQIzBBABCAAdFiEEdTtuzytFj/PRnVaMHgoog5euc54FAlpIHewACgkQHgoog5eu +c57UWA/+MiDU/qmDuOEwlU+KOBTuSdq12OKwC5X6uGc9yB4A62a0ADvICKDKs2NH +t5GOpW72Mrfin++370vHJUcDJFQXkllFvh3Fmt94ibq3kKUi6Ialgc5vPpN8qzYN +FpzAzoS3PayzNxSYujqxhc7OHfYRWhwGRHPZkL14HnMHUvlhj5/o55KreNoimiys +2DWVT3bqgP7DWDRTikKyZpzjv5rraIAnVCP1aq273HBIIcltPwEjEKV6Kw6DAFOz +SGIMclWfxml1+VwRhPYw4o18eeUQzKhE2STEG9TtnznhmjmixolAMdyi1Z0aUSPI +SAGgQBvs+Zwtk7/5Uaq2/+7KD+8xbRnJX5pZrf9dIUeLnMrnwVq0uU2hGOR+8amh +XbuBkvN7tVl6oXy/O3N8Usm0j30ZkzJL5MdCm00Y4EhhutkBhdYTP8YGizUZQxY6 +EJN0E89fl7JgX1ZeAvcFzPokAOkG06hKI77EBlia1hnrYtb0tRtLb/xvBrYyhJ0U +xqiNJSdxGX4Cq30Rokk3EAAHAQ4pF6eF9aJ1FCyTUK4YbOzNLXGAApL8k7dqf6Y7 +vEE/1VcB1r50cxe3r8FE2vpglRIoPxbUBuqa+LUbBMH0mW80Y3iPIRPOoGLkSwYX +5fV9Tfsr14YSq2Dk312Iefu+KKu9ddy40RQqx+ZR5kLrUb4hqxKJAjMEEAEKAB0W +IQTYyhd265JlSR0Hzmf1Ruy+qAnLGAUCW0pvCwAKCRD1Ruy+qAnLGGU/D/9calOr +u5GRHyTvMD7urChcVkgAcdbJucAUuNsQQDt0uMooIGq6m9W4xs2/eXV37ELoEDOT +A3swWtAG+oPVXkoTT+CBcns2Yv1t6LMwXwMFRdEsOOEn977DOh219NsyxP6QlFmz +lVl8i/6lrdSDXklbVW3ylOgpmYbm8b4ATOpbizeUvFr5wuel20y2e9X5BDI26RpF +bg110frfkrGB9y9DBvhJKffpMZiVwjoEVoHI2rOj3q3QuPOtJjP5Zuk5dq2Nm/ab +sWSXQjAx86b6jv4D0TqHd/W31n3o0V6za5tG2Rkz34iKtRujkonmG7X7aNw605UK +lyA1xNIItbOZmGhgH2iLS+lA2Vc5zwZPRltakbEpZfGOlJo72mpjilw5rZXaG7NI +Z4kty3pEwwUu6NRa//14PFoHXpDGVjnCOiM3suyiAlbEIUX/YAp/1m0FhnqTcnba +T6HeW42dCNeh0wRJBk5OCDR0E8+BXNeAEIhT+nI7FHGqY9F6sAJDlwBE5ylYm2Gy +Nn9rUNcopopbmytyzNzabkSAvs96h39RXeGlTt/Bz3Y9EHKVTx3zpkU3pAjMX+Po +IM7kOGUZZcsNLG1pBO1V86+m+HYBfq9cqlB0QUi3SNtVG6JYapHoDcx1ePpbSq5V +NWEW9y2nKUMfdjRg/RlmXtWiuFlMVCo2aSDqS4kCMwQQAQoAHRYhBNjKF3brkmVJ +HQfOZ/VG7L6oCcsYBQJbSnPNAAoJEPVG7L6oCcsY064QAJJa0EoxujxCrPi7fN8p +JxkRFfZpwDCiXMiKRO9GODHwRgmLyXvMNj+N7DL4/yB1iKEGTFfO9Nur/xv6OZJP +1lPPq77qJE2OMy8M4RkKp8z/KPyHlpJSZPJNoSpfkNGLMtz/140H7rhF1tGDeflC +I22rXO3qvgRZaN7UhWV5102V+bVQA1ZU9RdjIM/aUvec3Q+ctO4fiTSIz65hKcDe +1gc1bpygvVGlxrLhmifQuxwp/gf9cecsbzEmyH7aJAewva60CBe1pLeFWdutQsCh +jFY6vot2o4CzoED/2PvrdGa+55zVHAZuca8iJKHVfDn8anaGB8JrloL7J+GDQNog +7L/Y29O0Jv8gsD1ikEo2spnr1z5wmhiZHonnMuAjDtBy2KIIlbC+iHExoEraDv9/ +Z8DtZm/RNF5vRfrQW68GHz6Tqn/pfgFfVQJilQx+qxtAafoiYqmrPD2qE5HJOT7b +G6rxUU66Oq0p1SEE1RAnhF3hTSPC4yzy6tCcIx0OoIfc4Y077vSYJQgnxQUQqb+d +XIE0zuCfB7P8VZ62lvO6n7vd75m3QY0okgHje/Qs4R+mwrVoMRcl8G/57kW67FNa +A6JfdMSOhVY8J4b8URiIPa0FAG4UcdN3/7g0RqgqB+sVlU7KQ4UdRi3jxQB/CuFf +ozSQfdvbKa2r5mo2zb2yLZlhiQIzBBMBCgAdFiEE2MoXduuSZUkdB85n9UbsvqgJ +yxgFAltKdQYACgkQ9UbsvqgJyxhwvA/+M8mEtODQ3SmTrV8IY/QgkFQ1A9PamUMT +2YhaB6ino4OyQ0g/ng0owiRnoGGKzg8zfYliJPEadjyT5myRQDJl5Fgb2leaaGoB +vB6xgEeO5firV4xZscziDIMHW1W8gcg/QjkSvbbLh1SlC7YNUU87bJZvTgH5C+u1 +2iAhn8hOM7BT241QeSQZNTuHq0wFyNr6P10Plavyl2fctBaABA1eSBjM3T/610sf +El/Nq2M9NQqTvyWuZQ/qfgM9PlFsgoJ1i2BHnRaHLzjc3qDrIqODKkLvgDnRG1FF +5oeO/P8qssmxua3I9gVC+aedUZEl3Us6bv7MkVILL++cPqbDaqatS3jGXrbUzcbc +uXwZ4pEZPs0pX5dxncCxh8THSFgK7UwnUs5ihTliktr3xmoqgFvS3sAAWRuY4LPr +Lz5QLErE2AL512WRrS8EQr6DbMC/9k9mBO2Vst1+Vp+c94yBBLdEv9mu/Ca1T0rE +8kdjE0bBYLJnZjQZ16B91Xyj4TefF6dWjXWred7kfEiNPD8lxp6nkjeOxqHwFEEy +X4pSnmpG8U97JKYtW1lluIH9ND49K2PVBwbtxhyV2BB+zbDzn+NCr877BYNuDzGy +Ae2Vb/Zly5bqnBAVGxkqpSwsH3L4HM2gUBavWMQ8FUyHp4ZnQZOkheDD+VGaZUgV +trEUOeNyqJ+JAjYEMAEKACAWIQTYyhd265JlSR0Hzmf1Ruy+qAnLGAUCW0p1SQId +AAAKCRD1Ruy+qAnLGG3YD/4xX88pLFDrOEJoiIt28bzRET6IjCmh/+53AO7EgjhO +AztcJgdQQhdrTGODAWu8Lnjz7t2YsA9KE24fkvxwo5Q99Oz5NszyixgC/lMHQtyW +keiOxhoCcWSZ7nR9/t4IdcDWwWiMKrjHydIC+viRjWmRnq3eQ0DW6fZwtCLZj7Vl +Zh8JXSOL/jHhDvNow0IzSU9DAqoCmgM5vvAw9azJU8Z4BbrQKQKEvvY6AJPB2lHs +HLNT19ldT7Lm+6+Q6PHUy5wjZfJeso7TdPAS6khibiJY6zfNKuTW+cDxLrqhGX/B +uaZHtEO9aDRvmVC9oC4NJzAuyHLQuqzDfyk9Ob3Rt+9/BnNWXCtID0v168TBTWN9 +HWANuHxKfWwjrQBr5Xm4l8HzneMRmDUfkP8xSS5RcW/t1JHl3OQz6fGZZf4l5JMU +BJe0t9xRIlgEeBeIRC/ZrVNPqS4oGdsNpRO2TpuqLfNypg8cGFcAUHYHu/Wuev13 +cRBCeXvG1moFu0YXy2ESVIKqS+laOrBYb02hf+bhwf2jd8mKKA4eSiBXVbizCcpm +8QCLXhwBxXzZhHF7nc+kxyNvhzlpVuY9+PzqOK82ypTYDS+EY/FDJxfgDxm5ecOE +qooTePGmznMuMyS16rrPT/klFCTh9SV7HSOHH+co9LtB656tiNogQ2n0NsSG831J +UIkCNgQwAQoAIBYhBNjKF3brkmVJHQfOZ/VG7L6oCcsYBQJbSnVTAh0AAAoJEPVG +7L6oCcsYPacP/j3nzoxd6tgMhhb/jmN9Y9Tp1kOcajek4uIrwgtWIHODxljN6T5N +3JifClfKKkr9HGGw19ZGd4Oiajoz/vFNDVMF8OegGurDAIk2byaA8yN95vOrL2tr +k34AzIdOZT0kHl1iOuOXa4fRzb1J5R9/sdQNZPvnoAQ8YzDpJLIZbAwm94tbiyLZ +4chVNeWqKDwdxmK11Wq6h/Sg9tLSQstY9lOxsBfYOGbg7muYAiJ3Xr8TWGiBkvZY +FCQQ+PphR2w+GT43Py4J7I5W2jtRDz6k+K3hGj5MkqAk7cKknULQZjA5H1ZP/nWJ +LVa7/iLB4sIDtChlSxVBasSKGoAoZYX8Tu2EVPo+fL7GEvs2XqM97tWCsG/NOiDN +sUSo8T/tA39nMA9A2X+qSiry812w7+nXZxDhZQr2sHdwdbTQ080Qh9Xs4IKe8/sd +lq7mygUGnztAkSjVJalI9/NEv9yUt/R11Vpo6qEfAB2ADVhLoqkosOXr5NTvnnPX +dCgqqx0XjOte3EnJozV0P95dT8nCzAF3URDwxsNKVFN4Dg39zyMq8Gsihdps/V/b +XhIuSXCAQJdMqbhqErlE+dfp49FU2SmDMpw8L4nSbyb3beNCUn4ASI5TxWtc7jKg +YRRx7/b7PhvJMYzB5gsbKsd5juxFsYU2mKzEurZaNCVjjV3S2PY43IfMiQI5BBMB +AgAjBQJZ1qSvAhsBBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AACgkQkwOzOjBS +JMt1IhAAhhzn5g25u0pUcvgE+LJ14pdldwfHGi5gN9BZZ2l0X3ht7Hp6GdJwr7xL +vRoCq7d9oP3CicPUG1isuMoogx2gTHPkQ5F/mPpaudETCHpzB3Ucozdpdd4CxO86 +dypF/L6RFXcnRpwo3dU66RKA/Tmz/KgKhD0Byn+mx0Eq/ACnb2/lnzj21v+uorRq +U3s7cN8Coq2by8GUqkXjS+TLLFefthOfI5lM8XLki7qCHlwMmeroU97MviYVlpui +OOm9h7KAOBiHSz8cEcjFt8Dc0MjVmxf6PpFGD6DGeLN42KBNzXXnnev27IgnUiNo +IC2r/O7Z6pufwFN8m2aCAiURJtkQ0KhRaKESfISjvmYaf3xzoVJyuiKMcO8tiuAh +xX5f3wS3Y6QaKTmrU8ts0kLmdiHmvGYfd8WQysRLpQLNbetoSntY546cES+5KxIH +m1vu3jnfMJErD7xlAosPAkbsb4HMuFjmJvQQZbLDe3GnX0ZzrrimDnZeMr7ffn41 +JRglEatyfiyeckeo+L2alu2UdzJYxmtIqbo8O0bCmZEbOslLHTexc4xtdyhRLXzf +jpSBVpVkcT6yvlAsNZW7sNnp5lngOjcPrdNLkxbprNrAmdZNhIjd76tEe9fkBSdA +EaN6iGEUMVCDrfDEjMyxnFjoBehZDn0edmyzS6A+3ptRgteUz/uJAjkEEwEKACMW +IQSThqL7LanQ0x+vCBjAwHYTL/p2lQUCWfyOMAWDA8JnAAAKCRDAwHYTL/p2lbNa +EACM/cyjNhB5VxycBFP70g7D6xhFV9xYFA+lkp4E+OVUF4+guN4oaSWAgHs/o9BP +x4blaXdzyLmXwF+ZR00ye1PwU/UzNEPph9QXHlTgDB9lgr8pbv2foub1kMJ6STbK ++Z/f7E1EXZ/P2BVeyqbvGyVZwDZdo92068hnNvoV+Wk39bhv/2WzJlditj6hySeW +ExnvMlzgUXT9kWGq2Mn5RZvB1gNeAVM4Zu3PTgjm1IVs78c2NfNs6loJ0bJfTtm8 +LPfJ/p4uxPjHUAgaIr5byUyZD5udun1tMpSexu/0qjF2gy3dKCGg6sZXvuka8inG +xjzxsDpyjt7xXpRDRlRitXve4RrjO6vcU869P8DI9yMzloffuNlnhenU1+E4VizS +2Hb9CUugy04CufIUyYtuXCsA8hc7BJbfGbuuM0q3jHtrOoB+5PLYbEzydHTOkuhy +R4f9cUy32A33nO8yrZP6HqSTfUetJWoQCjUz7G5xNg4tM7cahc76YQfhQj6bezP/ +7SKNvvbSX1DUIc/1xFC7Mi7VcRXM6NlNxMIGGwu/rrmSyqRlEyF+LsrI2JJ1O8a4 +1mrBeEOeLy/sFv8zq7LyunAVENxif5g9vS5C4MAu//iMuno0fb6eqSaY9SBl4NhA +YcG5YycTJjaDHrEmqi0eoky5xxM76MHsQggfsAO/ZAbuQYkCPgQTAQIAKAIbAQIe +AQIXgAUCWjvlRAoLCQ0IDAcLCgMCBxUKCQgLAgMFFgIDAQAACgkQkwOzOjBSJMt6 +RA/9H6yJi3sb9WEv290OZStBGI2YgyyvpBVdOMNO7AXxPtKhUWIYOqDhR5Z+GQJd +4AYVR3l0ASdxWgBkN4CZCMhPHlrDfLyq9F1kZefK/xcoLn4s3NJh31orzep2jMuA +fVwp8SwvyWRmAhvlOkEJiSUcjqdamSwN5zVT3AN/OJcMldkdT6zZQSeyqkbZ/7SH +rVpSBwHD52iajP132bZImuQq6ABhTVcEyIyzIMyQeNA9UvLsEksVVhFXA84kS8aQ +kzuxYLAsGIe+zZtGFSscy20wuBp2FJJoQPBypv2utN5/YEqLzn/wX6YQx7W5B4Ra +GIUlEdc6jC/p+ET7/eF6dboVAuyB0ia+/Xh4ecSP4fG23Y7njruhL+BpKYaHoJfC +Cf4mAHcW0YAnrAa87mswKuUHbJ2u6PebG1FqQLZzw6vn1gy6+wCsj45cZqKGkG1/ +DCxECrfM5tDLXZD09X5XJ2e3ha0LDOajIfim8SEu5RkXrv2ODy2HAcWy0sHpyTkp +9MZ4BwkN2NmGroiKpHxXbZYP1BzycG5+jU/cKpSz9Hz22HsUT4Yx4We16o2srZF3 +srkRofByb3CuP3P97kEJ23s+7PrKJ2ygMIEqaTrQjSxXTdsiLayd/cYExKMcTbzt +WOp0Lp5+zZqa6hI5Y99tbPBk75g0AoC2MgKRDFza4d9YGrWJAnAEEAEIAFoWIQST +hqL7LanQ0x+vCBjAwHYTL/p2lQUCXDt4fAWDB4TOADYaaHR0cHM6Ly9qb25hdGhh +bmNyb3NzLmNvbS9DMEMwNzYxMzJGRkE3Njk1LnBvbGljeS50eHQACgkQwMB2Ey/6 +dpX+RhAAkyjGHy1umSuodZ/Pe2MTKF1689JzU5EsooLnJ+1E18MZQFHremRS5CEG +zEb7Z+FSFsmDkreITjLZFe9vvgzf6J8TyAX1NCAdarT1tdAhR1o26BWP1oj4eSIx +fycj+JLIOX4QJQlnAQ1hw0nfH3mFdEKRS7QKm9C1jz4g3/egcb/wcGiint5/NwvH +Pzl2OZgYDqvYgYiLMJDDZ405sIYQA9hdMrYRUVPcUOqWI8wbA+SXbyrjVZU3fqrX +P6nujZTwKE0Qg8dAfzAAdhT3LEl2ECP1gykT2PBYfQQ/3ZGbJKrjXhpFc7xlS4rr +ENfBqGtoc01Y9/NCGV3FTMOfT4LCEairQpnj2y7+PtKJzGFIyx3NkLPpli5lnO2p +wec+omFp/WA4jnLh3QLguwLCCAbcb/emchmw7zjC8ZAXC5+tN4z185wITZmZQv7w +GbYjIK+wtAF2DZ9tfTpjs3eDIO6WXA02ejivsa1lLIRPBsYEoBM7wRoSAQ+jN8Ww +3Z4zsSTq7sXzka6l00P42DW0TOtiMtXbzRk46elZ9Iq0Ck/YMXJwspwDbS/yqxrl +YI/vk9R3GY8kvudP0wMxlHvjyQooCMeq9vCN+JSBq5vlz1rjpNk0UwS2fLR517KK +kbGAPk12iAQycoIvimuOgPFmari3HVk3LufeUTUkpv6zLRL2kui5Ag0EWdamtwEQ +ANCxF+Zw1fdQ1tEkIoOh8JjVomVIMAa3sFkoHpZECjAICwa0v9EWSDGaKfZPcRA5 +PVddNjYwXHTSmch7uelkxP/r2dJZMHOihNZ2YsJtu+CxFg6SxHqyXVu6yCWRema7 +vZJhUy+/Zy+FsoOKo3q191dMRNLZKZYfhu/Yy19iWRVdsaDItWxyKODdut6fMn4b +nx9EUkswAyYTx7CNBDB/JFvPc8442fyVDPNxFbkpsN0rllYSexDncW0ZQBxj5BtS +iyFRdtkXjIGq+IXu8vnAx8kIS8tVs80dGuMZ92pjggHiPWyPEoQst3Crv2y3CgpC +BQVx4fQuViZZoKLL4M8y5cTXeYIq8XbPTo2v8RLVm52Q0zH8A6Qp0oQ0oXiXyh36 +/yoqdJWhPSqKIJ5Qtup7P1cHNccspS8zttJXyRiCouNxVu0VUDsjQzIAvQfol9yA +yp8VkWPHISyGm0PnOb/N3h6eeNxzw70pT0mBxGsJDMGjtN7Q7hoIzdzrg7Aap2TU +TVwlxtBdlrDGutgMKgEiBoSUBODK8FZZWhsxKtCeza3rLzaVJ6pAlGjq26y4Q2G6 +PbAxZfl2DaCa3yqX1hlWTjZ/7YVuS3TlZGiFZ14pygh/TVId2aERIxnv5t2qY0Iw +/0uoVWF/cfABth5DT/DId87rgHFLaTiFMLdeVeE3x2VBABEBAAGJBEQEGAECAA8F +AlnWprcCGwIFCQPCZwACKQkQkwOzOjBSJMvBXSAEGQECAAYFAlnWprcACgkQm3m0 +VpHbQXP4xQ//XClCycXfLWISSCahuKAYToWq300dBsvcPP6Gn3nO0ujiimXXwXeD +6pGS92kIvzaqiLYzqTHjlkKB50NqAgF49sUQzcnEKoFQVeh3EE/Uc4yNZr14nzyC +QdAQgpoFPy3rwCIhF5Dg+Un1xb3DKpwJ4sQlnU4HLHGqu3pQyqYoWSQ5gvYnltAC +HM2T02PDv6+5/04sg9S/zFJF+ECKa57qaqjZbKTw4G8CYCqbM02Q+CDGmPuehe7s +t+58BBa+z46J74b+tFjTYaqOQJr6K1ZTQ86SkP1Xj0W9KDiZwlWP+gzaadp03RhM +ZiyQ02Pn5FkQFNDCHo/MwwLZUSsKGN/nP+0ZG4hofTUnMRipGMgpM6c9JdAgfLtX +4Ip/Lh3dk724fl3I77pyIoXoLlh6sUSeQiuXu2ZW+bsoDe1NQRd16MGYzA4EvW0g +yxtXUSRlT42zihHFM7W6sTYdDHVe9MxiZ/H2gL0uuIfziNbewFb2BWYnXcYz1TVH +ez3FwQ7etQXaV2B4dBGQThvR24EeKA/5DnJqUGYNTX14KGOFPziK+CbFz+7OLo0p +25saDu/n8VAdVNx93Q0WnWnIw9qBBjovaomhsCDeMM7cxfvYKweEJh1FaIyyv73u +UHtBjtTLjTHaOPaK3OCMuy/Mdn/Sb9SIupMK1B3q9roeWGPMnl3U/4pqUA//VMUS +CTX97DyHYb/HCBiE8jC0B7VdmS+96lq8vhtwvD6mUzZRhlVvoOmx9VeoofizuI5V +Ecy9WDznb4iEig1ncuPvsrT3TMXSG8WPQcTV/GS30eevtxEDIGF66r5cWK1bkA18 +x8iKei9RXUufRqb4z56nTiFk1D0RVaaV+SSrTRPzAFUIFvPrBXXl/lvKDlyWh9ed +Ik8DRiCmEz1nA1XyeZEPArTDWZ0oLOJiyj1e5p3LFmQy01XhM3QILv/JABXUeaB4 +FFYMQJ8Y3twkI6TnwY8mJYh6fvcdCbB4/y1G2I82V7mRwAnlcEpVVUWwVifqxQv1 +bIQJMBBK1Ejo44RhQnrIPa7hXyuJU9VywtGN3ZgX7PE4ayAIJL7wBraIohu/dGVA +TWOYju2WuKtAFqvS4jN+O0FMfCyhmcEOAeS7oJUWoq2skTxs/h/BqbCPZNpKgaNr +oQ9hHWmMopfvIQ0jQxv1lAwZZF1V0m8B74lJQdRrwO/jZ9FDwXm9qqeNHskmKBbZ +wvzSrHjPVMWdY0qAWR+xq5aSqCd/e2ElqYX84vmRXlMOL+fCQlAqfiCqZP4PcsGM +niUgWOWAaJRYR1+sHSJq/K1zcTO0tgD8njKl1s5ywHV3yoO7rXu5worN+MxFpTLU +VBgjP6gGbrsiO3eFLP+KIY/aYn2rV0Q1poHIZfeJBFsEGAECACYCGwIWIQSo/FXz +sEujFG80kueTA7M6MFIkywUCXT2ShQUJBylSzgIpwV0gBBkBAgAGBQJZ1qa3AAoJ +EJt5tFaR20Fz+MUP/1wpQsnF3y1iEkgmobigGE6Fqt9NHQbL3Dz+hp95ztLo4opl +18F3g+qRkvdpCL82qoi2M6kx45ZCgedDagIBePbFEM3JxCqBUFXodxBP1HOMjWa9 +eJ88gkHQEIKaBT8t68AiIReQ4PlJ9cW9wyqcCeLEJZ1OByxxqrt6UMqmKFkkOYL2 +J5bQAhzNk9Njw7+vuf9OLIPUv8xSRfhAimue6mqo2Wyk8OBvAmAqmzNNkPggxpj7 +noXu7LfufAQWvs+Oie+G/rRY02GqjkCa+itWU0POkpD9V49FvSg4mcJVj/oM2mna +dN0YTGYskNNj5+RZEBTQwh6PzMMC2VErChjf5z/tGRuIaH01JzEYqRjIKTOnPSXQ +IHy7V+CKfy4d3ZO9uH5dyO+6ciKF6C5YerFEnkIrl7tmVvm7KA3tTUEXdejBmMwO +BL1tIMsbV1EkZU+Ns4oRxTO1urE2HQx1XvTMYmfx9oC9LriH84jW3sBW9gVmJ13G +M9U1R3s9xcEO3rUF2ldgeHQRkE4b0duBHigP+Q5yalBmDU19eChjhT84ivgmxc/u +zi6NKdubGg7v5/FQHVTcfd0NFp1pyMPagQY6L2qJobAg3jDO3MX72CsHhCYdRWiM +sr+97lB7QY7Uy40x2jj2itzgjLsvzHZ/0m/UiLqTCtQd6va6HlhjzJ5d1P+KCRCT +A7M6MFIky0CVD/0UbWHgtgboeK+iI81aJvuFSKi8ArUX+P2yOlCa1nEz8DqgZgw+ +99ik0qJIqxX/WsRmv9k86vLhSEGJaaizwbtH9QSlN7Nw5GCDPlW4pwYR84/bqUGI +aCE8s37IlZOYKmw5og9SldOH7Wx9+QSUX94BKhGbWoAjoNUBHK/ZFP91OJwwjlwQ +vzi1AWI4R40Ac8oGcS5beEBnr28Q3AkP0gGGbRUoqKEcNVa8ni2+DbjqIsh9k8Ot +x6393VRUmzeh+G3rpW5cbdDGei9xA6VdYN3nuhGxDKQilmaxCuz4Fht4RBmraVrk +/XhUO4PFKXu7Qr9tfK+KXJAjgoqI7tlSwKQWGZHX+wx/0+BIaEOSybxhbutW/rga +epFF8smIpwp+/eaNdnNjYj7DsasUy6EXAdV7OtoVrESTTLvozM0C/Ic2JGlXmgKI +CQv466Jkaq2LVqJ86/HwvWvwtxn+RFPSoEjOTXcGYvIuCbkhcyK8/6B28dQFF3El +Ml4Uzn3MmOTGceXk3KArfELtSuVyibcCi0OkeeT+b42x61AfWsvCosKP2YEqibzs +mJ/WG7odXkoqx/vzNWeobLCtPacg7hogO3OkumDN5OrLpbqzFj10bUwoZb9mZpAR +ZjB9v114n2MeKWA+Fdt0kEsk3iKppPMHvXj5Zc8/jk50HOZaw9uaFNfFh7kCDQRZ +1qw/ARAA2cbFunWAy38F7+JyTToKMd9PkjWakxgqyBUMK8AalRyNmHoIQVMX0mAT +KYf+oko7PPbwWkhRYKjuXDSyjyj3k+oppd8lNhztdiwaDhIzAVzMxYfE+Xd2tNsl +9ug9t8Ad6NSLfjOzRAx3JzNtiaQjudBFuprsUESpCYWZvNXB0hdOrnqM+XISNfNB +ep4N1ssuYWtiaQ4WPsVf2pfAutgpQJQgqe/X3H2JQZFz1lQjI+GeIjOStNnZYlPV +ipltTMLHDtxCPjLwuHSJxAFrA8mZPz4IvCnCUsnXgEWD6j9YOqqFpzpRq/Z7FEws +CeyKjE4TvjM00lvtKXCSPBFS3ydSlQSo423X2d0MMKmFRUPtwBHcmR/tLX4r+fzw +j2+uriSc7guPIXaDCLYulK/ThyodEceygjtXyZ/H1une2N0xLBxxqz50a00nx63W +llWRTxEFuq4GzR7t+eyMPZoYBxCRD8l9ohsXxVmSsC1B6JtkaZX21BidVUk+49Ol +ZDgHCBxr8zP19cBBpCrVrGizeHqGt8Ykgt7ytTYzEUxfpbHVaCYN33pq4QW/jsCZ +Op+rBsZP03YdvDLFYktedIaCPnokfQcCnMXb/qokg2OT/uaipbv47rGJN4eNTJSg +NU2Qvrc56iSMJBrYH8ds1oqVlrJ7mUf0Rcl2gFGzIM5k09sZiKMAEQEAAYkCJQQY +AQIADwUCWdasPwIbDAUJA8JnAAAKCRCTA7M6MFIky+D5D/4iQoycz1RHC9rSIdLn +vpQU2zT0zDBoDYSSj93sL0r34Hu3Jjwtvr35LGiYVF9nipV1iS0TmtzVSjBECbTI ++o0UTlkZzaYxp5ZZhQG271tumYiijW4uA7+aIMXaO4GrcCSuDHz+JFSgskdN/f5P +o07swmWGUmKMaMoXZDddn8Yyl5w2Nh+9329lTsaFWireGUHdeszwCdTPRhtgNyco +rpy0PuR1tlwmOCwiACb5W4leBjHfVagBdEXGUWuuVDpZBzWkHff30R7LtFRl7/Ku +kHjUQsS1+Jilzf8FsbixRQcaIvCuxOiSw5wpEM5SaN64qDm9gi83rjNJe7J3zVr8 +PAECtKZGRo39jIONKnll8FBIBv7wliRqd39F9RKpw9FKo6Mbp4O5y70xmZezTcaw +KnSYET33OuuS+KqgR7AIYMARJse0sicWHE7xLCAjPQtyk4mvpZ3FlHp1BzYh1/Uf +J5DKgqkL6fQ8EoqZLSh/RjgkZFttSOZ4CkhOc0KdYQD+nxJDaEDQ2F0Xdaxvyqea +EshSAVIP7G60GHut8nCKbLyjR/0vQet810TeOLtebW7SEoFnTAdOOn26XgsaUNxo +UiSs4/S44UHnRiD7is+cNlR4kNEglJPT3oITpT2UQuy18pi6LECRg+8aTlmcNT2H +graEOnn1/geEKl2T1VQW+rN8sYkCPAQYAQIAJgIbDBYhBKj8VfOwS6MUbzSS55MD +szowUiTLBQJdPZK6BQkHKU17AAoJEJMDszowUiTLxNAP/2QSjT4XKo6M/2LcF20R +K04to0lXKWnOrjE8Jdp+DzKGYFiYK35rkz7SKJYNw+FOUcVtMb7flJ2grwUjHrA2 +QZh2lieq48lCKeh3XykqtEFTEMO1ntujgEL8m0KrrwDyHAFJMO46hyIVd/8eZUvf +nJNafnDE1OZCnD89ZZSJLGEf3utMz6mgM4TJgaYGEDad7D+Xy/GuAHhaWVRziJHY +mULF/M8UjCRWvU5F/nRvyh+GMgXb2i0TwrL4VMI/ZTVo88rD1TuvA7a1aJMDdGTo +5dXFOGd15i+NYJYfF/eOVmkmCo39/YqfMQDuevUfwQJazviR4x3YAPLEfS8z3ryn +IKKqr0YmLtr5srNhExjqz2zJNaA0c6/qIWMw+KqygPsOfI3sYWevYFkqupswkll2 +Q5v723yP1C89Un6v+dhu354cJXKN8GxJ73/9tV4jsBB4rIJkzNWLaxkWKRZfYJZS +iQxEQkRIhXwRx+GUWdkXGN1WwzbhBR24CeuglT01ve5VkQ1EBl/XQ5+tdzsgiFmr +dT3RhyCpMipe59+ya5liJolFk2JEa9Ff71Zo5h2bbPkMlycSV/qYODoYdy6I2n2k +J/Bkt+pwRSMLaFsB7rXg+/Lt66DPKkUM0fQ6zmBXSCYMPMRgwUlq012jJokljeh3 +urPdZWtPdA8po7yLtcQH7JWbuQINBFnWrNgBEACw8HK2AB/Q935npfgzQrEBYpao +Eb5meJawnbfR8lUbIHRFCmjIaIgUysyNPiw+cM+OCi784dgpAqQOWVjjZC/j+tuD +SrHRZjr1LiaIfdADfHqoCq/U9GCAAjWWCn0niYa38tNtdiyp9Q/xaua/cqcsmGrE +oxubh17vZXA/E2fVJPh9KC0GU9t9pfK4MRi+kR0lEgx4SAEinbRIYNyvhiL6i3H+ +2LLfpAr0EfnSuCirNdbAHpdjWkfGNvp6EGOUKfmDz5E9tpLMjnjuwVyBngRrUfzk +Oi+L5ukcCOixMcw8ECJ05DExK3M8hPUXnWuxW1DTvj5L0OSEdM5yyyT7Zuw4Mz6k +0owEdTDbwlzULWPM1JstPBTbMmyS9C3A1/CdMgm6ON4tmoPhT+wp2FsvjEjPrLIt +LKD8/MObrkc+KLY03mkYFQKoFRFuSGtRkhNdIWDvezW9+y3deoGV2bKGMYNYSA7U +1JCuO/aI1k4OlQ4Dyd4tu/zas7OlVfBhuLodKUY8Q8qGDvCfu8xS55NXWj2xsv9d +Ss3yAYPriW7mhIA6CAeEsn8ZmKfkvBi+pnJbTqnoDniejbXFwZEUDhNS1zxrJGBW +2CYJHrP01E1npMqp1Cfh3cgvF50YJlxixXYaMrLMTYvoaPitBnT3L4agKBjFtGZK +V0tNpF7xJuWs/Dc0ZQARAQABiQIlBBgBAgAPBQJZ1qzYAhsgBQkDwmcAAAoJEJMD +szowUiTLDvYP/3PgdrKyhPtHbumWzZsP6Prscyu6fbFpSI4aE/c1b3qDIMTbA/Gh +qeqjf/tgnXb6/S+h9ke14L2qtQWT5pGBgcp8CjeNSveqDp7drqooRNsADzypIp4j +oD7LJoLLyKnX+DCYFMycpbkIEfsFVmjBEWtG63oW6UFwpdvvaLJXQwoAjOQuIJ4h +b/3KyfCRsQkHS8NKHvuDqWSSjsXtc2aMXKFYTIznsP6w4WK++myfLQH+v+wB+5o5 +Jqb/Fs1sb4OMI5New9kLUOFR2Z5ai1QbnmWev3Pyicd4mVNRAr2nH0bUIbfgP9fr +SjD4DxH7g1FW9Sv3XQNexKNsDAp68TPEx9ofWB0evCUE4dsawY86GEP+Somxz5YT +/GAJihFIIpFqYvSeNaYtjcGMkcWfqUTuHHPb48hE2nKyhVfTsHnHsUdV0kA24uTe +BG/8gELSrnVtgNvLWUvBp6/szsLeHynsCzBNZEwXWoKQ+AO3nSAccXKXfqtgyfro +Fh3qFaGYsyXrRguLNvQepou1OszqTET7HzrBEYMY3QgqGK1LVXpLL0V2ns8rQAlm +ip6xzHbWVx4NrzZJw8bhE6UpT6REL/DOU8u5TjntnCkXirwgCtjDlItYt7GRgEgc +h3G7yEfrOATyEBJyIk+z2M8YatYk49lG/ubDZ+AkgotAqwv91pt/EOh/iQI8BBgB +AgAmAhsgFiEEqPxV87BLoxRvNJLnkwOzOjBSJMsFAl09kscFCQcpTO8ACgkQkwOz +OjBSJMv0RBAAjOYHHv715G7r69qEDon2oM5td4We5JRt1xQWjq2AVsEnVPztsFi2 +5rexd0JzkLQYtHm7B/Yty/12LCvShR4IsU0NZwzkNAExU3kl0gbsiIihx7dd0L91 +z6wbYxYSG3o3+er8Rxq8ms4ac6I5IaEkeezsmmQ3OSG/iYfDw4yIquWOXI6mU63Q +CKGaS1rMnySq56sLHbT06cHK7n2PTn9BUVrqg7CQx0Kl+oIGrkBb7g0pj1Z6vlku +aV5QNWoDvA1tGhe+iZhcGQXSft0hlTFt4QFgr6Q5oCTEPcd4koRsTmkyZ3f2xDT+ +KIBiHtDVC5yveOz3WmQCP67mT439YFFf4/9eXtpwcZulp39pgycdkiZ/j0fRSh6U +sUbNLf6axked6TAPlguFE8+wrp1nJ3Il71VboCJvI5bwPt9+bYz8G9bSp8TIDkO5 +a0cciRQ4dmOA/uzimX1eHc1vS3o0OY5PK62r2QlfCU4nIqKB6an6bSbC8zqs7tSo +GKD340rU2Za0uBlXy8fVWNW4yKfwvLkxVDm/+hiqjMgmfSKAXocmSojZB2bs3tR3 +fpz90OdkDruOYVpGcRoW8HKUEnT10E0A/MdGw8ht1J9lNq8V8Poy2gywZr2dYoSz +vDG66CGMfVXJgGzFVshcnX2RycJhvyGS8YIIqYoE9k7Ww7BVYgVeYcM= +=abJr +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/xiphon.asc b/utils/gpg_keys/xiphon.asc new file mode 100644 index 000000000..01ad7fb77 --- /dev/null +++ b/utils/gpg_keys/xiphon.asc @@ -0,0 +1,41 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBF3UdnsBDACx+zgnCqDHg3hGPqtHr8J3QvOC04myIA1btztJRkqCaR7Saru0 +xe1E6HR8oMApboDn73u7HH6xcrELSjFwdbqs3ULDnurpc0CGg6ONqyfMjHTyOn/8 +8FKJioZNfcPrjooQg4ms3aYi6OZPErlX+6tyUtis9jE8Mb9A4mXyKKdN1zgL8ZRP +ygesZDW4TV5RTCYgHfUa6xYMJPuLCvRDU7/fP0wIByqLyWZUSmD8iENbNWUIMAMc +zU3HtfRPuHQgb7Vy5xxVR4ysaXEAUuzBqBKorngtg4tXPK4RfO06PHBunnGGENMP +zG0kpKoH9QRggLbIk5tkFGTodpFlH9ulx/w+mvJcZi58ZdMr5Nt6COaS5T9i1pj9 +42WDzXvfF1nAvww+VbYizh5bfdcMdcqx4gRrqXrQS7H++5RzghZsqy7/go9SV/Uq +ra617L0U8Wy9VSJVCoenh4i3WbKvpvLPb336yaz8LQafDrXTfVUZt9vkbD8qu4uH +ZxfEV/VYTdhi3JsAEQEAAbQeeGlwaG9uIDx4aXBob25AcHJvdG9ubWFpbC5jb20+ +iQHUBBMBCgA+FiEEj3eWTbujJBmNeKYDvXLsbD8YfGcFAl3UdnsCGwMFCQPCZwAF +CwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQvXLsbD8YfGc3mgv/fTCpmo1DzWEu +3kWNlYckfjOEsY7W1DSdED/pAwhqUZhVZ4KchQWRNsrOzNhESy3JaPPu30zEZDEY +Ixl+9i+2mFgwyu46VFYbJ2B+UWqr33hwBlgnrcwc0jDRFEtepqpnER95FwVmP5oW +KjyE/VJ68w/aYQDbylcDKIj6ILxK5nK7OjDk/pNKrXISnkQrGAAzdpgeAZO75OVE +hBH6oukAbsOVevLzXxovqR4vEeDLuAbB9jJqeZO8lltVw5NNu/HSeJGFk4fATJWE +k1+wHwKVl4AKt+AN1vQrJ++US3c7kAVTfoiAzIqoxXken09mm/O/+QIcXJON5A7I +7uZixh0NpLcADROrx+7+LarNDUSckGW8qtNl0vGBzNnKRVRKiAq6YllTOLNrpu5Q +MSZVUwpPxanCsXq3M7cyWlFQ0VM7YLqzKVkd6aSrz9RppSWLDfYJUwhRgRzUbP8x +4dV1pVX97LW//kPBndGTP33V13GOWEDjafT89WXrxzdHr/dmruu/uQGNBF3UdnsB +DADGNFzRFKhU3uuDgijavkYajePutFyhxY+LRmtHdYllAK8rKqzXx07cAv9N7HlR +vFq1B0MihMnJ48sXxfwan8AytOgf0zTVanvIr1dfx3c7A8yMuw6oZrmiT6ECcLNg +oKFM2IHtfLtrvz9KXobGpybcSrY6D46IzlIFUr6PfwJIqT1+hWsq9bagn8HeWEsw +7zxX1ZkYSGem7cbFG9xzIJSig82hkxXzfihWHNm3fYlpuKC2TKHYDnSkcshx4MwZ +DX3ubmu25VZ1HPlZNdUJtHpUG1zBtTLetc7IFRjdYN6wO2ttDtdrjF4A9ZQNZRmb +yYQaeV58cHgJs6dDycGpek7YvRom9Ueg1F4+mKlvLsY1JFxBfgyH3W2Lj3A1A/za +h2jZxrRXd0PeCEEKr+gsvJiyP8Irm7TMEsjw/assF8AyVorj+Fy5vqnPdelPkYS9 +Y9HDeusFhvtUBem4orBgubiVzjRr+5Pk4lWRhUppT6Riw1SCV9rwGb8gNrkrJU6h +ZkUAEQEAAYkBvAQYAQoAJhYhBI93lk27oyQZjXimA71y7Gw/GHxnBQJd1HZ7AhsM +BQkDwmcAAAoJEL1y7Gw/GHxn+1QL/1FlcIQOVfNkj4GxKg1qrdHmTn24Qibf/aMA +kyN2l5i2lynFE9Bu7nWcdoorsrxtXrsdGu/WiP/89h/yzUh1CFcPa/kwN7/KCNiH +URK6rLtJiGpTJC3HaPxQdudZk0gacVdtgTy441UIF8WCcWFLD0Nq7qTc8VxHWqhu +ow3nr63234Mqf+GvQ49lV5x2vkmRycBwqNxpxv8O04r3ux6dVH8HlQ6Rzomj2ILd +86YLzPNhHo3XvPTdE3LOzB/M/H4sdxbb+r98FblIqGcZHj4RJyRIIkPjvBeL44QL +1huxGhE2BfCMLU9gHyPSRcKo+6qVnFuTtSjSjW1gT2gGtmG5OOQR/PvEa+Zxfwg6 +X3NyO66IypknfhjCLTPUnT5E0/5ZrpFra/kFIdHzOPpG3eBEOiCAeAbizsXHOiHR +K2cf3XM8WF7vIDEgNHlH4kqq85wIJQakeC8VN5JjcMT5Q255WHo2bYDob4HeR4dH +JezqaExCLJDmDVnOj/t8V7dUv4o84A== +=y+nZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/logs/levin-traffic.awk b/utils/logs/levin-traffic.awk new file mode 100755 index 000000000..25c1c3ed1 --- /dev/null +++ b/utils/logs/levin-traffic.awk @@ -0,0 +1,73 @@ +#!/bin/awk -f + +function max(a, b) { if (a < b) return b; return a; } +function bytes_str(b) { if (b < 1024) return b " bytes"; if (b < 1024 * 1024) return b/1024 " kB"; return b/1024/1024 " MB"; } +function time_str(b) { if (b < 120) return b " sec"; if (b < 3600) return b/60 " min"; if (b < 86400) return b/3600 " hours"; return b/86400 " days"} + +BEGIN { +commands["command-1001"] = "HANDSHAKE" +commands["command-1002"] = "TIMED_SYNC" +commands["command-1003"] = "PING" +commands["command-1004"] = "REQUEST_STAT_INFO" +commands["command-1005"] = "REQUEST_NETWORK_STATE" +commands["command-1006"] = "REQUEST_PEER_ID" +commands["command-1007"] = "REQUEST_SUPPORT_FLAGS" +commands["command-2001"] = "NOTIFY_NEW_BLOCK" +commands["command-2002"] = "NOTIFY_NEW_TRANSACTIONS" +commands["command-2003"] = "REQUEST_GET_OBJECTS" +commands["command-2004"] = "RESPONSE_GET_OBJECTS" +commands["command-2006"] = "NOTIFY_REQUEST_CHAIN" +commands["command-2007"] = "RESPONSE_CHAIN_ENTRY" +commands["command-2008"] = "NOTIFY_NEW_FLUFFY_BLOCK" +commands["command-2009"] = "NOTIFY_REQUEST_FLUFFY_MISSING_TX" +} + +/ net.p2p.traffic / { + date=gensub(/-/, " ", "g", $1) + time=gensub(/\..*/, "", "g", gensub(/:/, " ", "g", $2)) + ip=gensub(/\[/, "", "g", $7) + outin=gensub(/]/, "", "g", $8) + timestamp=date " " time + timestamp=mktime(timestamp) + if (!t0) + t0 = timestamp + if (!t0ip[ip]) + t0ip[ip] = timestamp + t1 = timestamp + t1ip[ip] = timestamp + bytes=$9 + dir=$11 + command=$14 + initiator=$17 + + bytes_by_command[command] += bytes + bytes_by_ip[ip] += bytes + if (dir == "sent") + bytes_sent_by_ip[ip] += bytes + else + bytes_received_by_ip[ip] += bytes + bytes_by_outin[outin] += bytes + bytes_by_direction[dir] += bytes + bytes_by_initiator[initiator] += bytes +} + +END { + dt = t1 - t0 + print "Running time:", time_str(dt) + for (command in bytes_by_command) { + category = command + if (commands[command]) + category = commands[command]; + print "Category", category ":", bytes_str(bytes_by_command[command]) + } + for (direction in bytes_by_direction) print direction ":", bytes_str(bytes_by_direction[direction]) + for (initiator in bytes_by_initiator) print "Initiating from", initiator ":", bytes_str(bytes_by_initiator[initiator]) + for (outin in bytes_by_outin) print "With", outin, "peers:", bytes_str(bytes_by_outin[outin]) + for (ip in bytes_by_ip) print "IP", ip ":", bytes_str(bytes_by_ip[ip]) + print "Download rate:", bytes_str(bytes_by_direction["received"] / max(dt, 1)) "/s" + for (ip in bytes_received_by_ip) + print " ", ip ":", bytes_str(bytes_received_by_ip[ip] / max(t1ip[ip] - t0ip[ip], 1)) "/s over", time_str(t1ip[ip] - t0ip[ip]) + print "Upload rate:", bytes_str(bytes_by_direction["sent"] / max(dt, 1)) "/s" + for (ip in bytes_sent_by_ip) + print " ", ip ":", bytes_str(bytes_sent_by_ip[ip] / max(t1ip[ip] - t0ip[ip], 1)) "/s over", time_str(t1ip[ip] - t0ip[ip]) +} diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index 6a3fabdc9..ac9ba2d3a 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -237,14 +237,15 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(create_account) - def create_address(self, account_index = 0, label = ""): + def create_address(self, account_index = 0, label = "", count = 1): create_address = { 'method': 'create_address', 'params' : { 'account_index': account_index, - 'label': label + 'label': label, + 'count': count }, - 'jsonrpc': '2.0', + 'jsonrpc': '2.0', 'id': '0' } return self.rpc.send_json_rpc_request(create_address) @@ -705,11 +706,13 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(check_reserve_proof) - def sign(self, data): + def sign(self, data, account_index = 0, address_index = 0): sign = { 'method': 'sign', 'params' : { 'data': data, + 'account_index': account_index, + 'address_index': address_index, }, 'jsonrpc': '2.0', 'id': '0' @@ -838,11 +841,12 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(validate_address) - def get_accounts(self, tag): + def get_accounts(self, tag, strict_balances = False): get_accounts = { 'method': 'get_accounts', 'params': { 'tag': tag, + 'strict_balances': strict_balances, }, 'jsonrpc': '2.0', 'id': '0' @@ -1062,6 +1066,20 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(stop_mining) + def estimate_tx_size_and_weight(self, n_inputs, n_outputs, ring_size = 0, rct = True): + estimate_tx_size_and_weight = { + 'method': 'estimate_tx_size_and_weight', + 'jsonrpc': '2.0', + 'params': { + 'n_inputs': n_inputs, + 'n_outputs': n_outputs, + 'ring_size': ring_size, + 'rct': rct, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(estimate_tx_size_and_weight) + def get_version(self): get_version = { 'method': 'get_version', diff --git a/utils/translations/build-translations.sh b/utils/translations/build-translations.sh index 1217dca0a..c868a691f 100755 --- a/utils/translations/build-translations.sh +++ b/utils/translations/build-translations.sh @@ -7,6 +7,10 @@ then fi if test -z "$lrelease" then + lrelease=`which lrelease-qt5 2> /dev/null` +fi +if test -z "$lrelease" +then echo "lrelease not found" exit 1 fi @@ -17,7 +21,7 @@ then languages="" for language in $(cat translations/ready) do - languages="$languages translations/$language.ts" + languages="$languages translations/monero_$language.ts" done else languages="translations/*.ts" |