diff options
92 files changed, 1878 insertions, 655 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3c552dd8..b97be58b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,19 +6,20 @@ on: [push, pull_request] env: REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost BUILD_DEFAULT_LINUX: | - ccache --max-size=150M cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release && cmake --build build -j3 APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache' APT_SET_CONF: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + CCACHE_SETTINGS: | + ccache --max-size=150M + ccache --set-config=compression=true jobs: build-macos: runs-on: macOS-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 @@ -30,16 +31,15 @@ jobs: key: ccache-${{ runner.os }}-build-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build- - name: install dependencies - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} make -j3 build-windows: runs-on: windows-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp CCACHE_DIR: C:\Users\runneradmin\.ccache defaults: @@ -60,7 +60,7 @@ jobs: install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} make release-static-win64 -j2 # See the OS labels and monitor deprecations here: @@ -69,7 +69,6 @@ jobs: build-ubuntu: runs-on: ${{ matrix.os }} env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp strategy: matrix: @@ -92,12 +91,13 @@ jobs: - name: install monero dependencies run: ${{env.APT_INSTALL_LINUX}} - name: build - run: ${{env.BUILD_DEFAULT_LINUX}} + run: | + ${{env.CCACHE_SETTINGS}} + ${{env.BUILD_DEFAULT_LINUX}} libwallet-ubuntu: runs-on: ubuntu-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 @@ -118,7 +118,7 @@ jobs: run: ${{env.APT_INSTALL_LINUX}} - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} cmake . make wallet_api -j3 @@ -126,7 +126,6 @@ jobs: needs: build-ubuntu runs-on: ubuntu-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 @@ -152,6 +151,7 @@ jobs: env: CTEST_OUTPUT_ON_FAILURE: ON run: | + ${{env.CCACHE_SETTINGS}} ${{env.BUILD_DEFAULT_LINUX}} cmake --build build --target test diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index 8e4eaf177..74d0ceabc 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -2,11 +2,19 @@ name: ci/gh-actions/depends on: [push, pull_request] +env: + APT_SET_CONF: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + CCACHE_SETTINGS: | + ccache --max-size=150M + ccache --set-config=compression=true + jobs: build-macos: runs-on: ubuntu-18.04 env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp strategy: fail-fast: false @@ -69,10 +77,7 @@ jobs: key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} restore-keys: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: install dependencies run: sudo apt update; sudo apt -y install build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache ${{ matrix.toolchain.packages }} - name: prepare apple-darwin11 @@ -88,7 +93,7 @@ jobs: sudo update-alternatives --set ${{ matrix.toolchain.host }}-gcc $(which ${{ matrix.toolchain.host }}-gcc-posix) - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} make depends target=${{ matrix.toolchain.host }} -j2 - uses: actions/upload-artifact@v2 if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b9ba570d..f71133ec0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,14 @@ endif() enable_language(C ASM) +# Require C11/C++11 and disable extensions for all targets +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + function (die msg) if (NOT WIN32) string(ASCII 27 Esc) @@ -384,18 +392,6 @@ endif() message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) - -if (NOT DEFINED ENV{DEVELOPER_LIBUNBOUND_OLD}) - message(STATUS "Could not find DEVELOPER_LIBUNBOUND_OLD in env (not required)") -elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 1) - message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 1, will use the work around") - add_definitions(-DDEVELOPER_LIBUNBOUND_OLD) -elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 0) - message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 0") -else() - message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD with bad value. Will NOT use the work around") -endif() - set_property(GLOBAL PROPERTY USE_FOLDERS ON) enable_testing() @@ -503,6 +499,7 @@ if(STATIC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DZMQ_STATIC") endif() +option(SANITIZE "Use ASAN memory sanitizer" OFF) if(SANITIZE) if (MSVC) message(FATAL_ERROR "Cannot sanitize with MSVC") @@ -571,7 +568,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "(SunOS|Solaris)") endif () if (APPLE AND NOT IOS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") if (NOT OPENSSL_ROOT_DIR) EXECUTE_PROCESS(COMMAND brew --prefix openssl OUTPUT_VARIABLE OPENSSL_ROOT_DIR @@ -619,8 +616,6 @@ if (NOT (MSVC OR ARCH)) set_default_arch() endif() -CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) - option(COVERAGE "Enable profiling for test coverage report" OFF) if(COVERAGE) message(STATUS "Building with profiling for test coverage report") @@ -793,14 +788,14 @@ else() endif() set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes") set(CXX_WARNINGS "-Wno-reorder -Wno-missing-field-initializers") - try_compile(STATIC_ASSERT_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.c" COMPILE_DEFINITIONS "-std=c11") + try_compile(STATIC_ASSERT_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.c" CMAKE_FLAGS -DCMAKE_C_STANDARD=11) if(STATIC_ASSERT_RES) set(STATIC_ASSERT_FLAG "") else() set(STATIC_ASSERT_FLAG "-Dstatic_assert=_Static_assert") endif() - try_compile(STATIC_ASSERT_CPP_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.cpp" COMPILE_DEFINITIONS "-std=c++11") + try_compile(STATIC_ASSERT_CPP_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.cpp" CMAKE_FLAGS -DCMAKE_CXX_STANDARD=11) if(STATIC_ASSERT_CPP_RES) set(STATIC_ASSERT_CPP_FLAG "") else() @@ -896,8 +891,8 @@ else() message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}") # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that @@ -1001,11 +996,6 @@ else() endif() set(DEBUG_FLAGS "-g3") - if(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT (CMAKE_C_COMPILER_VERSION VERSION_LESS 4.8)) - set(DEBUG_FLAGS "${DEBUG_FLAGS} -Og ") - else() - set(DEBUG_FLAGS "${DEBUG_FLAGS} -O0 ") - endif() # At least some CLANGs default to not enough for monero set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=900") @@ -1242,10 +1232,6 @@ endif() # when ON - will install libwallet_merged into "lib" option(BUILD_GUI_DEPS "Build GUI dependencies." OFF) -# This is not nice, distribution packagers should not enable this, but depend -# on libunbound shipped with their distribution instead -option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source vendored with this repo." OFF) - find_package(PythonInterp) find_program(iwyu_tool_path NAMES iwyu_tool.py iwyu_tool) if (iwyu_tool_path AND PYTHONINTERP_FOUND) diff --git a/Dockerfile b/Dockerfile index f21f74ac3..587007470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,192 +1,45 @@ # Multistage docker build, requires docker 17.05 # builder stage -FROM ubuntu:16.04 as builder +FROM ubuntu:20.04 as builder RUN set -ex && \ apt-get update && \ - apt-get --no-install-recommends --yes install \ + DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends --yes install \ + automake \ + autotools-dev \ + bsdmainutils \ + build-essential \ ca-certificates \ + ccache \ cmake \ - g++ \ - make \ - pkg-config \ - graphviz \ - doxygen \ - git \ curl \ - libtool-bin \ - autoconf \ - automake \ - bzip2 \ - xsltproc \ - gperf \ - unzip - -WORKDIR /usr/local - -ENV CFLAGS='-fPIC' -ENV CXXFLAGS='-fPIC' - -#Cmake -ARG CMAKE_VERSION=3.14.6 -ARG CMAKE_VERSION_DOT=v3.14 -ARG CMAKE_HASH=4e8ea11cabe459308671b476469eace1622e770317a15951d7b55a82ccaaccb9 -RUN set -ex \ - && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \ - && echo "${CMAKE_HASH} cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf cmake-${CMAKE_VERSION}.tar.gz \ - && cd cmake-${CMAKE_VERSION} \ - && ./configure \ - && make \ - && make install - -## Boost -ARG BOOST_VERSION=1_70_0 -ARG BOOST_VERSION_DOT=1.70.0 -ARG BOOST_HASH=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 -RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOT}/boost_${BOOST_VERSION}.tar.bz2 \ - && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ - && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ - && cd boost_${BOOST_VERSION} \ - && ./bootstrap.sh \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="$CFLAGS" cxxflags="$CXXFLAGS" stage -ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} - -# OpenSSL -ARG OPENSSL_VERSION=1.1.1i -ARG OPENSSL_HASH=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 -RUN set -ex \ - && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ - && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ - && cd openssl-${OPENSSL_VERSION} \ - && ./Configure linux-x86_64 no-shared --static "$CFLAGS" \ - && make build_generated \ - && make libcrypto.a \ - && make install -ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION} - -# ZMQ -ARG ZMQ_VERSION=v4.3.2 -ARG ZMQ_HASH=a84ffa12b2eb3569ced199660bac5ad128bff1f0 -RUN set -ex \ - && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ - && cd libzmq \ - && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure --enable-static --disable-shared \ - && make \ - && make install \ - && ldconfig - -# zmq.hpp -ARG CPPZMQ_VERSION=v4.4.1 -ARG CPPZMQ_HASH=f5b36e563598d48fcc0d82e589d3596afef945ae -RUN set -ex \ - && git clone https://github.com/zeromq/cppzmq.git -b ${CPPZMQ_VERSION} \ - && cd cppzmq \ - && test `git rev-parse HEAD` = ${CPPZMQ_HASH} || exit 1 \ - && mv *.hpp /usr/local/include - -# Readline -ARG READLINE_VERSION=8.0 -ARG READLINE_HASH=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 -RUN set -ex \ - && curl -s -O https://ftp.gnu.org/gnu/readline/readline-${READLINE_VERSION}.tar.gz \ - && echo "${READLINE_HASH} readline-${READLINE_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf readline-${READLINE_VERSION}.tar.gz \ - && cd readline-${READLINE_VERSION} \ - && ./configure \ - && make \ - && make install - -# Sodium -ARG SODIUM_VERSION=1.0.18 -ARG SODIUM_HASH=4f5e89fa84ce1d178a6765b8b46f2b6f91216677 -RUN set -ex \ - && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ - && cd libsodium \ - && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure \ - && make \ - && make check \ - && make install - -# Udev -ARG UDEV_VERSION=v3.2.8 -ARG UDEV_HASH=d69f3f28348123ab7fa0ebac63ec2fd16800c5e0 -RUN set -ex \ - && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ - && cd eudev \ - && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ - && make \ - && make install - -# Libusb -ARG USB_VERSION=v1.0.22 -ARG USB_HASH=0034b2afdcdb1614e78edaa2a9e22d5936aeae5d -RUN set -ex \ - && git clone https://github.com/libusb/libusb.git -b ${USB_VERSION} \ - && cd libusb \ - && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure --disable-shared \ - && make \ - && make install - -# Hidapi -ARG HIDAPI_VERSION=hidapi-0.8.0-rc1 -ARG HIDAPI_HASH=40cf516139b5b61e30d9403a48db23d8f915f52c -RUN set -ex \ - && git clone https://github.com/signal11/hidapi -b ${HIDAPI_VERSION} \ - && cd hidapi \ - && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ - && ./bootstrap \ - && ./configure --enable-static --disable-shared \ - && make \ - && make install - -# Protobuf -ARG PROTOBUF_VERSION=v3.7.1 -ARG PROTOBUF_HASH=6973c3a5041636c1d8dc5f7f6c8c1f3c15bc63d6 -RUN set -ex \ - && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ - && cd protobuf \ - && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \ - && git submodule update --init --recursive \ - && ./autogen.sh \ - && ./configure --enable-static --disable-shared \ - && make \ - && make install \ - && ldconfig + git \ + libtool \ + pkg-config \ + gperf WORKDIR /src COPY . . -ENV USE_SINGLE_BUILDDIR=1 ARG NPROC RUN set -ex && \ git submodule init && git submodule update && \ rm -rf build && \ if [ -z "$NPROC" ] ; \ - then make -j$(nproc) release-static ; \ - else make -j$NPROC release-static ; \ + then make -j$(nproc) depends target=x86_64-linux-gnu ; \ + else make -j$NPROC depends target=x86_64-linux-gnu ; \ fi # runtime stage -FROM ubuntu:16.04 +FROM ubuntu:20.04 RUN set -ex && \ apt-get update && \ apt-get --no-install-recommends --yes install ca-certificates && \ apt-get clean && \ rm -rf /var/lib/apt -COPY --from=builder /src/build/release/bin /usr/local/bin/ +COPY --from=builder /src/build/x86_64-linux-gnu/release/bin /usr/local/bin/ # Create monero user RUN adduser --system --group --disabled-password monero && \ @@ -805,7 +805,7 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = */build/* */contrib/depends/* # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the @@ -123,12 +123,12 @@ release-static-linux-armv7: release-static-android-armv7: mkdir -p $(builddir)/release/translations cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE) - cd $(builddir)/release && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android-armv7" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARM_MODE=ON -D CMAKE_ANDROID_ARCH_ABI="armeabi-v7a" ../.. && $(MAKE) + cd $(builddir)/release && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-armv7" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARM_MODE=ON -D CMAKE_ANDROID_ARCH_ABI="armeabi-v7a" ../.. && $(MAKE) release-static-android-armv8: mkdir -p $(builddir)/release/translations cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE) - cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE) + cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE) release-static-linux-armv8: mkdir -p $(builddir)/release @@ -42,12 +42,9 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Research -The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. +The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with the Monero research community if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. -Our researchers are available on IRC in [#monero-research-lab on Libera](https://web.libera.chat/#monero-research-lab) or by email: - -- Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) -- Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) +The Monero research community is available on IRC in [#monero-research-lab on Libera](https://web.libera.chat/#monero-research-lab), which is also accessible via Matrix. ## Announcements @@ -132,8 +129,8 @@ Dates are provided in the format YYYY-MM-DD. | 1686275 | 2018-10-19 | v9 | v0.13.0.0 | v0.13.0.4 | bulletproofs required | 1788000 | 2019-03-09 | v10 | v0.14.0.0 | v0.14.1.2 | New PoW based on Cryptonight-R, new block weight algorithm, slightly more efficient RingCT format | 1788720 | 2019-03-10 | v11 | v0.14.0.0 | v0.14.1.2 | forbid old RingCT transaction format -| 1978433 | 2019-11-30* | v12 | v0.15.0.0 | v0.16.0.0 | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs -| 2210000 | 2020-10-17 | v13 | v0.17.0.0 | v0.17.1.1 | New CLSAG transaction format +| 1978433 | 2019-11-30 | v12 | v0.15.0.0 | v0.16.0.0 | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs +| 2210000 | 2020-10-17 | v13 | v0.17.0.0 | v0.17.1.1 | New CLSAG transaction format | 2210720 | 2020-10-18 | v14 | v0.17.1.1 | v0.17.1.7 | forbid old MLSAG transaction format | XXXXXXX | XXX-XX-XX | XXX | vX.XX.X.X | vX.XX.X.X | XXX | @@ -160,7 +157,7 @@ library archives (`.a`). | Dep | Min. version | Vendored | Debian/Ubuntu pkg | Arch pkg | Void pkg | Fedora pkg | Optional | Purpose | | ------------ | ------------- | -------- | -------------------- | ------------ | ------------------ | ------------------- | -------- | --------------- | -| GCC | 4.7.3 | NO | `build-essential` | `base-devel` | `base-devel` | `gcc` | NO | | +| GCC | 5 | NO | `build-essential` | `base-devel` | `base-devel` | `gcc` | NO | | | CMake | 3.5 | NO | `cmake` | `cmake` | `cmake` | `cmake` | NO | | | pkg-config | any | NO | `pkg-config` | `base-devel` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | `boost-devel` | NO | C++ libraries | @@ -187,7 +184,14 @@ library archives (`.a`). | libudev | ? | No | `libudev-dev` | `systemd` | `eudev-libudev-devel` | `systemd-devel` | YES | Hardware wallet | [1] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must -build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ``` +build the library binary manually. This can be done with the following command `sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make` +then: + +* on Debian: + `sudo mv libg* /usr/lib/` +* on Ubuntu: + `sudo mv lib/libg* /usr/lib/` + [2] libnorm-dev is needed if your zmq library was built with libnorm, and not needed otherwise Install all dependencies at once on Debian/Ubuntu: diff --git a/cmake/GitVersion.cmake b/cmake/GitVersion.cmake index 6a62bcaa3..2839d549f 100644 --- a/cmake/GitVersion.cmake +++ b/cmake/GitVersion.cmake @@ -32,7 +32,7 @@ function (get_version_tag_from_git GIT) execute_process(COMMAND "${GIT}" rev-parse --short=9 HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -49,7 +49,7 @@ function (get_version_tag_from_git GIT) # Get all the tags execute_process(COMMAND "${GIT}" tag -l --points-at HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE TAG OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/contrib/depends/packages/hidapi.mk b/contrib/depends/packages/hidapi.mk index b76ef1548..8c56f9187 100644 --- a/contrib/depends/packages/hidapi.mk +++ b/contrib/depends/packages/hidapi.mk @@ -1,9 +1,10 @@ package=hidapi -$(package)_version=0.9.0 +$(package)_version=0.11.0 $(package)_download_path=https://github.com/libusb/hidapi/archive $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=630ee1834bdd5c5761ab079fd04f463a89585df8fcae51a7bfe4229b1e02a652 +$(package)_sha256_hash=391d8e52f2d6a5cf76e2b0c079cfefe25497ba1d4659131297081fc0cd744632 $(package)_linux_dependencies=libusb eudev +$(package)_patches=missing_win_include.patch define $(package)_set_vars $(package)_config_opts=--enable-static --disable-shared @@ -16,6 +17,10 @@ $(package)_config_opts_linux+=libusb_CFLAGS=-I$(host_prefix)/include/libusb-1.0 $(package)_config_opts_linux+=--with-pic endef +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/missing_win_include.patch +endef + define $(package)_config_cmds ./bootstrap &&\ $($(package)_autoconf) $($(package)_config_opts) AR_FLAGS=$($(package)_arflags) diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index d80775b22..0dddca072 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,9 @@ package=openssl -$(package)_version=1.1.1k +$(package)_version=1.1.1l $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 +$(package)_sha256_hash=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 +$(package)_patches=fix_darwin.patch define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" @@ -51,7 +52,8 @@ endef define $(package)_preprocess_cmds sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \ - sed -i -e 's|cflags --sysroot.*",|cflags",|' Configurations/15-android.conf + sed -i -e 's|cflags --sysroot.*",|cflags",|' Configurations/15-android.conf && \ + patch -p1 < $($(package)_patch_dir)/fix_darwin.patch endef define $(package)_config_cmds @@ -59,11 +61,11 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) -j1 build_libs libcrypto.pc libssl.pc openssl.pc + $(MAKE) build_libs endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw + $(MAKE) DESTDIR=$($(package)_staging_dir) install_sw endef define $(package)_postprocess_cmds diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 56ce425bb..eed9e8ec1 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,6 +1,9 @@ packages:=boost openssl zeromq libiconv expat ldns unbound +# ccache is useless in gitian builds +ifneq ($(GITIAN),1) native_packages := native_ccache +endif hardware_packages := hidapi protobuf libusb hardware_native_packages := native_protobuf @@ -8,8 +11,8 @@ hardware_native_packages := native_protobuf android_native_packages = android_ndk android_packages = ncurses readline sodium -darwin_native_packages = native_biplist native_ds_store native_mac_alias $(hardware_native_packages) -darwin_packages = sodium ncurses readline $(hardware_packages) +darwin_native_packages = $(hardware_native_packages) +darwin_packages = ncurses readline sodium $(hardware_packages) # not really native... freebsd_native_packages = freebsd_base @@ -31,6 +34,6 @@ mingw32_packages = icu4c sodium $(hardware_packages) mingw32_native_packages = $(hardware_native_packages) ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus +darwin_native_packages += native_cctools endif diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk index 826a820c4..c3d190bca 100644 --- a/contrib/depends/packages/unwind.mk +++ b/contrib/depends/packages/unwind.mk @@ -1,8 +1,8 @@ package=unwind -$(package)_version=1.2 +$(package)_version=1.5.0 $(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind $(package)_file_name=lib$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992 +$(package)_sha256_hash=90337653d92d4a13de590781371c604f9031cdb50520366aa1e3a91e1efb1017 $(package)_patches=fix_obj_order.patch define $(package)_preprocess_cmds @@ -12,7 +12,7 @@ endef define $(package)_config_cmds cp -f $(BASEDIR)/config.guess config/config.guess &&\ cp -f $(BASEDIR)/config.sub config/config.sub &&\ - $($(package)_autoconf) --disable-shared --enable-static AR_FLAGS=$($(package)_arflags) + $($(package)_autoconf) --disable-shared --enable-static --disable-tests --disable-documentation AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/zeromq.mk b/contrib/depends/packages/zeromq.mk index 55941e67d..20f2d4bd9 100644 --- a/contrib/depends/packages/zeromq.mk +++ b/contrib/depends/packages/zeromq.mk @@ -1,9 +1,9 @@ package=zeromq -$(package)_version=4.1.7 -$(package)_download_path=https://github.com/zeromq/zeromq4-1/releases/download/v$($(package)_version)/ +$(package)_version=4.3.4 +$(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=31c383cfcd3be1dc8a66e448c403029e793687e70473b89c4cc0bd626e7da299 -$(package)_patches=9114d3957725acd34aa8b8d011585812f3369411.patch 9e6745c12e0b100cd38acecc16ce7db02905e27c.patch ffe62d3398d5e0191f554f61049aa7ec9fc892ae.patch +$(package)_sha256_hash=c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5 +$(package)_patches=06aba27b04c5822cb88a69677382a0f053367143.patch define $(package)_set_vars $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve @@ -13,10 +13,7 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/9114d3957725acd34aa8b8d011585812f3369411.patch && \ - patch -p1 < $($(package)_patch_dir)/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch && \ - patch -p1 < $($(package)_patch_dir)/ffe62d3398d5e0191f554f61049aa7ec9fc892ae.patch && \ - ./autogen.sh + patch -p1 < $($(package)_patch_dir)/06aba27b04c5822cb88a69677382a0f053367143.patch endef define $(package)_config_cmds @@ -24,7 +21,7 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) libzmq.la + $(MAKE) src/libzmq.la endef define $(package)_stage_cmds diff --git a/contrib/depends/patches/hidapi/missing_win_include.patch b/contrib/depends/patches/hidapi/missing_win_include.patch new file mode 100644 index 000000000..5bbe82def --- /dev/null +++ b/contrib/depends/patches/hidapi/missing_win_include.patch @@ -0,0 +1,21 @@ +From a77b066311da42ed7654e39c0356a3b951b2e296 Mon Sep 17 00:00:00 2001 +From: selsta <selsta@sent.at> +Date: Wed, 10 Nov 2021 02:28:54 +0100 +Subject: [PATCH] windows: add missing include for mingw32 + +--- + windows/hid.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/windows/hid.c b/windows/hid.c +index 24756a4..6d8394c 100644 +--- a/windows/hid.c ++++ b/windows/hid.c +@@ -33,6 +33,7 @@ typedef LONG NTSTATUS; + #endif + + #ifdef __MINGW32__ ++#include <devpropdef.h> + #include <ntdef.h> + #include <winbase.h> + #endif diff --git a/contrib/depends/patches/openssl/fix_darwin.patch b/contrib/depends/patches/openssl/fix_darwin.patch new file mode 100644 index 000000000..a917daa12 --- /dev/null +++ b/contrib/depends/patches/openssl/fix_darwin.patch @@ -0,0 +1,60 @@ +From 96ac8f13f4d0ee96baf5724d9f96c44c34b8606c Mon Sep 17 00:00:00 2001 +From: David Carlier <devnexen@gmail.com> +Date: Tue, 24 Aug 2021 22:40:14 +0100 +Subject: [PATCH] Darwin platform allows to build on releases before + Yosemite/ios 8. + +issue #16407 #16408 + +Reviewed-by: Paul Dale <pauli@openssl.org> +Reviewed-by: Tomas Mraz <tomas@openssl.org> +(Merged from https://github.com/openssl/openssl/pull/16409) +--- + crypto/rand/rand_unix.c | 5 +---- + include/crypto/rand.h | 10 ++++++++++ + 2 files changed, 11 insertions(+), 4 deletions(-) + +diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c +index 43f1069d151d..0f4525106af7 100644 +--- a/crypto/rand/rand_unix.c ++++ b/crypto/rand/rand_unix.c +@@ -34,9 +34,6 @@ + #if defined(__OpenBSD__) + # include <sys/param.h> + #endif +-#if defined(__APPLE__) +-# include <CommonCrypto/CommonRandom.h> +-#endif + + #if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__) + # include <sys/types.h> +@@ -381,7 +378,7 @@ static ssize_t syscall_random(void *buf, size_t buflen) + if (errno != ENOSYS) + return -1; + } +-# elif defined(__APPLE__) ++# elif defined(OPENSSL_APPLE_CRYPTO_RANDOM) + if (CCRandomGenerateBytes(buf, buflen) == kCCSuccess) + return (ssize_t)buflen; + +diff --git a/include/crypto/rand.h b/include/crypto/rand.h +index 5350d3a93119..674f840fd13c 100644 +--- a/include/crypto/rand.h ++++ b/include/crypto/rand.h +@@ -20,6 +20,16 @@ + + # include <openssl/rand.h> + ++# if defined(__APPLE__) && !defined(OPENSSL_NO_APPLE_CRYPTO_RANDOM) ++# include <Availability.h> ++# if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101000) || \ ++ (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 80000) ++# define OPENSSL_APPLE_CRYPTO_RANDOM 1 ++# include <CommonCrypto/CommonCryptoError.h> ++# include <CommonCrypto/CommonRandom.h> ++# endif ++# endif ++ + /* forward declaration */ + typedef struct rand_pool_st RAND_POOL; + diff --git a/contrib/depends/patches/unwind/fix_obj_order.patch b/contrib/depends/patches/unwind/fix_obj_order.patch index 374a9f04a..e764f0f3d 100644 --- a/contrib/depends/patches/unwind/fix_obj_order.patch +++ b/contrib/depends/patches/unwind/fix_obj_order.patch @@ -1,9 +1,9 @@ ---- config/ltmain.sh.O 2017-01-13 16:00:54.000000000 +0000 -+++ config/ltmain.sh 2019-11-17 06:46:51.994402494 +0000 -@@ -7957,6 +7957,8 @@ - esac - done +--- config/ltmain.sh.0 2020-11-10 17:25:26.000000000 +0100 ++++ config/ltmain.sh 2021-09-11 19:39:36.000000000 +0200 +@@ -10768,6 +10768,8 @@ fi + func_to_tool_file "$oldlib" func_convert_file_msys_to_w32 + tool_oldlib=$func_to_tool_file_result + oldobjs=`for obj in $oldobjs; do echo $obj; done | sort` + oldobjs=" `echo $oldobjs`" eval cmds=\"$old_archive_cmds\" diff --git a/contrib/depends/patches/zeromq/06aba27b04c5822cb88a69677382a0f053367143.patch b/contrib/depends/patches/zeromq/06aba27b04c5822cb88a69677382a0f053367143.patch new file mode 100644 index 000000000..53e18a452 --- /dev/null +++ b/contrib/depends/patches/zeromq/06aba27b04c5822cb88a69677382a0f053367143.patch @@ -0,0 +1,22 @@ +From 06aba27b04c5822cb88a69677382a0f053367143 Mon Sep 17 00:00:00 2001 +From: sabotagebeats <27985126+sabotagebeats@users.noreply.github.com> +Date: Thu, 22 Jul 2021 21:53:19 -0700 +Subject: [PATCH] fix: building libzmq fails with error src/clock.cpp:131:16: + error: unused variable 'nsecs_per_usec' + +--- + src/clock.cpp | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/clock.cpp b/src/clock.cpp +index 93da90a8e..63c0100a5 100644 +--- a/src/clock.cpp ++++ b/src/clock.cpp +@@ -195,6 +195,7 @@ uint64_t zmq::clock_t::now_us () + + #else + ++ LIBZMQ_UNUSED (nsecs_per_usec); + // Use POSIX gettimeofday function to get precise time. + struct timeval tv; + int rc = gettimeofday (&tv, NULL); diff --git a/contrib/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch b/contrib/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch deleted file mode 100644 index f704b3d94..000000000 --- a/contrib/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 9114d3957725acd34aa8b8d011585812f3369411 Mon Sep 17 00:00:00 2001 -From: Jeroen Ooms <jeroenooms@gmail.com> -Date: Tue, 20 Oct 2015 13:10:38 +0200 -Subject: [PATCH] enable static libraries on mingw - ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 393505b..e92131a 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -265,7 +265,7 @@ case "${host_os}" in - libzmq_dso_visibility="no" - - if test "x$enable_static" = "xyes"; then -- AC_MSG_ERROR([Building static libraries is not supported under MinGW32]) -+ CPPFLAGS="-DZMQ_STATIC" - fi - - # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/contrib/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch b/contrib/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch deleted file mode 100644 index 9aff2c179..000000000 --- a/contrib/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 9e6745c12e0b100cd38acecc16ce7db02905e27c Mon Sep 17 00:00:00 2001 -From: David Millard <dmillard10@gmail.com> -Date: Tue, 10 May 2016 13:53:53 -0700 -Subject: [PATCH] Fix autotools for static MinGW builds - ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 5a0fa14..def6ea7 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -259,7 +259,7 @@ case "${host_os}" in - libzmq_dso_visibility="no" - - if test "x$enable_static" = "xyes"; then -- CPPFLAGS="-DZMQ_STATIC" -+ CPPFLAGS="-DZMQ_STATIC $CPPFLAGS" - fi - - # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/contrib/depends/patches/zeromq/ffe62d3398d5e0191f554f61049aa7ec9fc892ae.patch b/contrib/depends/patches/zeromq/ffe62d3398d5e0191f554f61049aa7ec9fc892ae.patch deleted file mode 100644 index a532df1b6..000000000 --- a/contrib/depends/patches/zeromq/ffe62d3398d5e0191f554f61049aa7ec9fc892ae.patch +++ /dev/null @@ -1,38 +0,0 @@ -From ffe62d3398d5e0191f554f61049aa7ec9fc892ae Mon Sep 17 00:00:00 2001 -From: Gregory Lemercier <greglemercier@free.fr> -Date: Sun, 7 Oct 2018 18:06:54 +0200 -Subject: [PATCH] Fix build on arm64 architectures with some strict compilers - -This patch fixes an issue that occurs on 64-bit architetures under -strict compiler rules. The code initially checked that the received -size stored in 'uint64_t' was not bigger than the max value of a -'size_t' variable, which is legitimate on 32-bit architectures where -'size_t' variables are stored on 32 bits. On 64-bit architectures, -this test no longer makes sense since 'uint64_t' and 'size_t' types -have the same size. The issue is fixed by ignoring this portion -of code when built for arm64. ---- - src/v1_decoder.cpp | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/v1_decoder.cpp b/src/v1_decoder.cpp -index b002dc9d..2c8c97a7 100644 ---- a/src/v1_decoder.cpp -+++ b/src/v1_decoder.cpp -@@ -114,11 +114,13 @@ int zmq::v1_decoder_t::eight_byte_size_ready () - return -1; - } - -+#ifndef __aarch64__ - // Message size must fit within range of size_t data type. - if (payload_length - 1 > std::numeric_limits <size_t>::max ()) { - errno = EMSGSIZE; - return -1; - } -+#endif - - const size_t msg_size = static_cast <size_t> (payload_length - 1); - --- -2.20.1 - diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index a87b9c058..c3a0df59b 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -83,7 +83,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") SET(PORT OFF) SET(CMAKE_OSX_SYSROOT "@sdk@/MacOSX10.11.sdk/") SET(CMAKE_OSX_DEPLOYMENT_TARGET "10.08") - SET(CMAKE_CXX_STANDARD 11) + SET(CMAKE_CXX_STANDARD 14) SET(CMAKE_OSX_ARCHITECTURES "x86_64") SET(LLVM_ENABLE_PIC OFF) SET(LLVM_ENABLE_PIE OFF) diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 07f119ba4..3fa2f7fb4 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -47,9 +47,7 @@ if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) monero_add_library(epee_readline readline_buffer.cpp) endif() -if(HAVE_C11) -SET_PROPERTY(SOURCE memwipe.c PROPERTY COMPILE_FLAGS -std=c11) -endif() +set_property(SOURCE memwipe.c PROPERTY C_STANDARD 11) # Build and install libepee if we're building for GUI if (BUILD_GUI_DEPS) diff --git a/contrib/epee/src/buffer.cpp b/contrib/epee/src/buffer.cpp index 10ea6de56..cf94fb5d5 100644 --- a/contrib/epee/src/buffer.cpp +++ b/contrib/epee/src/buffer.cpp @@ -26,6 +26,7 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include <limits> #include <string.h> #include "net/buffer.h" diff --git a/contrib/epee/src/http_auth.cpp b/contrib/epee/src/http_auth.cpp index 18ee18bd4..9f30e1975 100644 --- a/contrib/epee/src/http_auth.cpp +++ b/contrib/epee/src/http_auth.cpp @@ -209,12 +209,7 @@ namespace } template<typename T> - using quoted_result = boost::joined_range< - const boost::joined_range<const boost::string_ref, const T>, const boost::string_ref - >; - - template<typename T> - quoted_result<T> quoted(const T& arg) + auto quoted_(const T& arg) // avoid ADL selecting C++14 std::quoted { return boost::range::join(boost::range::join(ceref(u8"\""), arg), ceref(u8"\"")); } @@ -242,13 +237,13 @@ namespace { str.append(u8"Digest "); add_first_field(str, u8"algorithm", algorithm); - add_field(str, u8"nonce", quoted(user.server.nonce)); - add_field(str, u8"realm", quoted(user.server.realm)); - add_field(str, u8"response", quoted(response)); - add_field(str, u8"uri", quoted(uri)); - add_field(str, u8"username", quoted(user.credentials.username)); + add_field(str, u8"nonce", quoted_(user.server.nonce)); + add_field(str, u8"realm", quoted_(user.server.realm)); + add_field(str, u8"response", quoted_(response)); + add_field(str, u8"uri", quoted_(uri)); + add_field(str, u8"username", quoted_(user.credentials.username)); if (!user.server.opaque.empty()) - add_field(str, u8"opaque", quoted(user.server.opaque)); + add_field(str, u8"opaque", quoted_(user.server.opaque)); } //! Implements superseded algorithm specified in RFC 2069 @@ -674,8 +669,8 @@ namespace Digest::name, (i == 0 ? boost::string_ref{} : sess_algo) ); add_field(out, u8"algorithm", algorithm); - add_field(out, u8"realm", quoted(auth_realm)); - add_field(out, u8"nonce", quoted(nonce)); + add_field(out, u8"realm", quoted_(auth_realm)); + add_field(out, u8"nonce", quoted_(nonce)); add_field(out, u8"stale", is_stale ? ceref("true") : ceref("false")); fields.push_back(std::make_pair(std::string(server_auth_field), std::move(out))); diff --git a/contrib/epee/src/wipeable_string.cpp b/contrib/epee/src/wipeable_string.cpp index 21312d49d..03cb05232 100644 --- a/contrib/epee/src/wipeable_string.cpp +++ b/contrib/epee/src/wipeable_string.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/optional/optional.hpp> +#include <limits> #include <string.h> #include "memwipe.h" #include "misc_log_ex.h" diff --git a/contrib/epee/tests/src/CMakeLists.txt b/contrib/epee/tests/src/CMakeLists.txt index 0c42f87ca..e724b53f4 100644 --- a/contrib/epee/tests/src/CMakeLists.txt +++ b/contrib/epee/tests/src/CMakeLists.txt @@ -1,6 +1,13 @@ cmake_minimum_required(VERSION 3.5) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + set(Boost_USE_MULTITHREADED ON) include_directories(.) @@ -14,8 +21,8 @@ IF (MSVC) include_directories(SYSTEM platform/msvc) ELSE() # set stuff for other systems - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -Wall") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-reorder") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-reorder") ENDIF() diff --git a/contrib/gitian/DOCKRUN.md b/contrib/gitian/DOCKRUN.md new file mode 100644 index 000000000..fa5e13c82 --- /dev/null +++ b/contrib/gitian/DOCKRUN.md @@ -0,0 +1,85 @@ +Quick Gitian building with docker +================================= + +*Setup instructions for a Gitian build of Monero using Docker.* + +Gitian supports other container mechanisms too but if you have a Debian or +Ubuntu-based host the steps can be greatly simplified. + +Preparing the Gitian builder host +--------------------------------- + +The procedure here will create a docker container for build preparation, as well as +for actually running the builds. The only items you must install on your own host +are docker and apt-cacher-ng. With docker installed, you should also give yourself +permission to use docker by adding yourself to the docker group. + +```bash +sudo apt-get install docker.io apt-cacher-ng +sudo usermod -aG docker $USER +su $USER +``` + +The final `su` command is needed to start a new shell with your new group membership, +since the `usermod` command doesn't affect any existing sessions. + +If you want Mac binaries included in your build, you need to obtain the MacOS SDK: + +```bash +curl -O https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz +``` + +Other User Preparation +---------------------- + +The final step will be to `gpg` sign the results of your build and upload them to GitHub. +Before you can do that, you'll need +* a GitHub account. +If your GitHub account name is different from your local account name, you must +set your GitHub account name for the script to use: + +```bash +export GH_USER=<github account name> +``` + +* PGP keys - if you don't have one already, you can use `gpg --quick-gen-key` to generate it. +* a fork of the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo on your GitHub account. +Please follow the directions there for uploading your key first. + +**Note:** Please ensure your gpg public key is available to check signatures by adding it to the [gitian.sigs/gitian-pubkeys/](https://github.com/monero-project/gitian.sigs/tree/master/gitian-pubkeys) directory in a pull request. + + +Building the Binaries +--------------------- + +The dockrun.sh script will do everything to build the binaries. Just specify the +version to build as its only argument, e.g. + +```bash +./dockrun.sh v0.17.2.3 +``` + +The build should run to completion with no errors, and will display the SHA256 checksums +of the resulting binaries. You'll be prompted to check if the sums look good, and if so +then the results will be signed, and the signatures will be pushed to GitHub. + +You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / GitHub to figure out what is going on. + + +Other Options +------------- + +This script just runs the [gitian-build.py](gitian-build.py) inside a container named `gitrun`. +You can set other options for that script by setting the OPT variable when running `dockrun.sh` +e.g. + +```bash +OPT="-j 8" ./dockrun.sh v0.17.2.3 +``` + +You can also examine the build and install logs by running a shell in the container, e.g. + +```bash +docker exec -it gitrun /bin/bash +more builder/var/install-linux.log +``` diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index 0a40d4608..c922a2373 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -31,6 +31,8 @@ This guide explains how to set up the environment, and how to start the builds. * Gitian gives you the option of using any of 3 different virtualization tools: `kvm`, `docker` or `lxc`. This documentation will only show how to build with `lxc` and `docker` (documentation for `kvm` is welcome). Building with `lxc` is the default, but is more complicated, so we recommend docker your first time. +* For a shortcut using `docker` follow the instructions in [DOCKRUN.md](DOCKRUN.md) instead +of following the rest of this document.. ## Create the gitianuser account diff --git a/contrib/gitian/dockrun.sh b/contrib/gitian/dockrun.sh new file mode 100755 index 000000000..015c411fd --- /dev/null +++ b/contrib/gitian/dockrun.sh @@ -0,0 +1,133 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "usage: $0 <version>" + exit 1 +fi +VERSION=$1 + +DOCKER=`command -v docker` +CACHER=`command -v apt-cacher-ng` + +if [ -z "$DOCKER" -o -z "$CACHER" ]; then + echo "$0: you must first install docker.io and apt-cacher-ng" + echo " e.g. sudo apt-get install docker.io apt-cacher-ng" + exit 1 +fi + +GH_USER=${GH_USER-$USER} + +TAG=gitrun-bionic +TAG2=base-bionic-amd64 +IMAGE=`docker images | grep $TAG` + +WORKDIR=/home/ubuntu + +if [ -z "$IMAGE" ]; then +GID=`getent group docker` +mkdir -p docker +cd docker + +# container for running gitian-build.py +cat <<EOF > ${TAG}.Dockerfile +FROM ubuntu:bionic + +ENV DEBIAN_FRONTEND=noninteractive +RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher +RUN echo "$GID" >> /etc/group +RUN apt-get update && apt-get --no-install-recommends -y install lsb-release ruby git make wget docker.io python3 curl + +RUN useradd -ms /bin/bash -U ubuntu -G docker +USER ubuntu:docker +WORKDIR $WORKDIR + +RUN git clone https://github.com/monero-project/gitian.sigs.git sigs; \ + git clone https://github.com/devrandom/gitian-builder.git builder; \ + cd builder; git checkout c0f77ca018cb5332bfd595e0aff0468f77542c23; mkdir -p inputs var; cd inputs; \ + git clone https://github.com/monero-project/monero + +CMD ["sleep", "infinity"] +EOF + +docker build --pull -f ${TAG}.Dockerfile -t $TAG . + +cd .. +docker run -v /var/run/docker.sock:/var/run/docker.sock -d --name gitrun $TAG +if [ -f MacOSX10.11.sdk.tar.gz ]; then + docker cp MacOSX10.11.sdk.tar.gz gitrun:$WORKDIR/builder/inputs/ +else + echo "No MacOS SDK found, Mac builds will be omitted" +fi + +fi + +IMAGE=`docker images | grep $TAG2` +if [ -z "$IMAGE" ]; then +mkdir -p docker +cd docker + +# container for actually running each build +cat <<EOF > ${TAG2}.Dockerfile +FROM ubuntu:bionic + +ENV DEBIAN_FRONTEND=noninteractive +RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher +RUN apt-get update && apt-get --no-install-recommends -y install build-essential git language-pack-en \ + wget lsb-release curl gcc-7 g++-7 gcc g++ binutils-gold pkg-config autoconf libtool automake faketime \ + bsdmainutils ca-certificates python cmake gperf + +RUN useradd -ms /bin/bash -U ubuntu +USER ubuntu:ubuntu +WORKDIR $WORKDIR + +CMD ["sleep", "infinity"] +EOF + +docker build --pull -f ${TAG2}.Dockerfile -t $TAG2 . + +cd .. + +fi + +RUNNING=`docker ps | grep gitrun` +if [ -z "$RUNNING" ]; then + BUILT=`docker ps -a | grep gitrun` + if [ -z "$BUILT" ]; then + docker run -v /var/run/docker.sock:/var/run/docker.sock -d --name gitrun $TAG + else + docker start gitrun + fi +fi +docker cp gitian-build.py gitrun:$WORKDIR/ +docker exec -t gitrun ./gitian-build.py -d -b -D -n $OPT $GH_USER $VERSION +RC=$? +if [ $RC != 0 ]; then + exit $RC +fi +echo "\nBuild Results:\n" +docker exec gitrun sh -c "sha256sum out/$VERSION/*" +echo "\nIf these results look correct, type \"sign\" to sign them, otherwise ^C to stop now." +read check +if [ "$check" != "sign" ]; then + echo "Not signing, bye." + exit 1 +fi + +if [ ! -d sigs ]; then + git clone https://github.com/monero-project/gitian.sigs.git sigs + cd sigs + git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git + cd .. +fi + +DIRS=`docker exec gitrun sh -c "echo sigs/$VERSION-*"` +for i in $DIRS; do + docker cp gitrun:$WORKDIR/$i sigs + gpg --detach-sign $i/$GH_USER/*.assert +done + +cd sigs +git checkout -b $VERSION +git add $VERSION-* +git commit -S -m "Add $GH_USER $VERSION" +git push --set-upstream $GH_USER $VERSION diff --git a/contrib/gitian/gitian-android.yml b/contrib/gitian/gitian-android.yml index b8eaa8af9..23cb7d0e8 100644 --- a/contrib/gitian/gitian-android.yml +++ b/contrib/gitian/gitian-android.yml @@ -24,12 +24,6 @@ packages: - "ca-certificates" - "python" - "cmake" -- "ccache" -- "protobuf-compiler" -- "libdbus-1-dev" -- "libharfbuzz-dev" -- "libprotobuf-dev" -- "python3-zmq" - "unzip" remotes: - "url": "https://github.com/monero-project/monero.git" @@ -52,6 +46,7 @@ script: | if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + export GITIAN=1 mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi @@ -125,8 +120,8 @@ script: | cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release make ${MAKEOPTS} chmod 755 bin/* - cp ../LICENSE bin - chmod 644 bin/LICENSE + cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin + chmod 644 bin/LICENSE bin/*.md DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 diff --git a/contrib/gitian/gitian-freebsd.yml b/contrib/gitian/gitian-freebsd.yml index 36b81c641..134823b95 100644 --- a/contrib/gitian/gitian-freebsd.yml +++ b/contrib/gitian/gitian-freebsd.yml @@ -25,12 +25,6 @@ packages: - "ca-certificates" - "python" - "cmake" -- "ccache" -- "protobuf-compiler" -- "libdbus-1-dev" -- "libharfbuzz-dev" -- "libprotobuf-dev" -- "python3-zmq" remotes: - "url": "https://github.com/monero-project/monero.git" "dir": "monero" @@ -52,6 +46,7 @@ script: | if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + export GITIAN=1 mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi @@ -122,8 +117,8 @@ script: | cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=ON make ${MAKEOPTS} chmod 755 bin/* - cp ../LICENSE bin - chmod 644 bin/LICENSE + cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin + chmod 644 bin/LICENSE bin/*.md DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 0aac983cc..7ab628fbc 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -36,12 +36,6 @@ packages: - "ca-certificates" - "python" - "cmake" -- "ccache" -- "protobuf-compiler" -- "libdbus-1-dev" -- "libharfbuzz-dev" -- "libprotobuf-dev" -- "python3-zmq" remotes: - "url": "https://github.com/monero-project/monero.git" "dir": "monero" @@ -63,6 +57,7 @@ script: | if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + export GITIAN=1 mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi @@ -167,8 +162,8 @@ script: | cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON make ${MAKEOPTS} chmod 755 bin/* - cp ../LICENSE bin - chmod 644 bin/LICENSE + cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin + chmod 644 bin/LICENSE bin/*.md DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 9889ca45f..3434861d2 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -41,6 +41,7 @@ script: | if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + export GITIAN=1 mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi @@ -111,8 +112,8 @@ script: | cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make ${MAKEOPTS} chmod 755 bin/* - cp ../LICENSE bin - chmod 644 bin/LICENSE + cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin + chmod 644 bin/LICENSE bin/*.md DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index c53086144..7d5a249c8 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -20,7 +20,6 @@ packages: - "zip" - "ca-certificates" - "python" -- "rename" - "cmake" alternatives: - @@ -54,6 +53,7 @@ script: | if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + export GITIAN=1 mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi @@ -127,7 +127,7 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make ${MAKEOPTS} - cp ../LICENSE bin + cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip diff --git a/docs/ZMQ.md b/docs/ZMQ.md index 9128ff2ad..a7006f486 100644 --- a/docs/ZMQ.md +++ b/docs/ZMQ.md @@ -25,6 +25,8 @@ allows for filtering on: (1) format, (2) context, and (3) event. Includes previously unseen transactions in a block but _not_ the `miner_tx`. Does not "re-publish" after a reorg. Includes `do_not_relay` transactions. + * `miner_data` - provides the necessary data to create a custom block template + Available only in the `full` context. The subscription topics are formatted as `format-context-event`, with prefix matching supported by both Monero and ZMQ. The `format`, `context` and `event` diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 9aa4c08bc..462f774bf 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -30,7 +30,10 @@ cmake_minimum_required(VERSION 3.5) project(easylogging CXX) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + monero_enable_coverage() find_package(Threads) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 267770074..a765ee8cc 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2984,8 +2984,8 @@ void Writer::initializeLogger(Logger *logger, bool needLock) { } void Writer::processDispatch() { - static std::atomic_flag in_dispatch; - if (in_dispatch.test_and_set()) + static __thread bool in_dispatch = false; + if (in_dispatch) { if (m_proceed && m_logger != NULL) { @@ -2994,6 +2994,7 @@ void Writer::processDispatch() { } return; } + in_dispatch = true; #if ELPP_LOGGING_ENABLED if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { bool firstDispatched = false; @@ -3032,7 +3033,7 @@ void Writer::processDispatch() { m_logger->releaseLock(); } #endif // ELPP_LOGGING_ENABLED - in_dispatch.clear(); + in_dispatch = false; } void Writer::triggerDispatch(void) { diff --git a/external/randomx b/external/randomx -Subproject fe4324e8c0c035fec3affd6e4c49241c2e5b995 +Subproject 9efc398c196ef1c50d8e6f5e1f2c5ac02f1f6ce diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 1d7b10648..de2700ad0 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -467,7 +467,12 @@ void mdb_txn_safe::allow_new_txns() creation_gate.clear(); } -void lmdb_resized(MDB_env *env) +void mdb_txn_safe::increment_txns(int i) +{ + num_active_txns += i; +} + +void lmdb_resized(MDB_env *env, int isactive) { mdb_txn_safe::prevent_new_txns(); @@ -478,7 +483,11 @@ void lmdb_resized(MDB_env *env) mdb_env_info(env, &mei); uint64_t old = mei.me_mapsize; + if (isactive) + mdb_txn_safe::increment_txns(-1); mdb_txn_safe::wait_no_active_txns(); + if (isactive) + mdb_txn_safe::increment_txns(1); int result = mdb_env_set_mapsize(env, 0); if (result) @@ -496,7 +505,7 @@ inline int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB { int res = mdb_txn_begin(env, parent, flags, txn); if (res == MDB_MAP_RESIZED) { - lmdb_resized(env); + lmdb_resized(env, 1); res = mdb_txn_begin(env, parent, flags, txn); } return res; @@ -506,7 +515,7 @@ inline int lmdb_txn_renew(MDB_txn *txn) { int res = mdb_txn_renew(txn); if (res == MDB_MAP_RESIZED) { - lmdb_resized(mdb_txn_env(txn)); + lmdb_resized(mdb_txn_env(txn), 0); res = mdb_txn_renew(txn); } return res; @@ -1267,11 +1276,11 @@ BlockchainLMDB::~BlockchainLMDB() // batch transaction shouldn't be active at this point. If it is, consider it aborted. if (m_batch_active) { - try { batch_abort(); } + try { BlockchainLMDB::batch_abort(); } catch (...) { /* ignore */ } } if (m_open) - close(); + BlockchainLMDB::close(); } BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() @@ -1569,9 +1578,9 @@ void BlockchainLMDB::close() if (m_batch_active) { LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); - batch_abort(); + BlockchainLMDB::batch_abort(); } - this->sync(); + BlockchainLMDB::sync(); m_tinfo.reset(); // FIXME: not yet thread safe!!! Use with care. @@ -1584,7 +1593,7 @@ void BlockchainLMDB::sync() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - if (is_read_only()) + if (BlockchainLMDB::is_read_only()) return; // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 0e6d70039..d87bc6e49 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -154,6 +154,7 @@ struct mdb_txn_safe static void prevent_new_txns(); static void wait_no_active_txns(); static void allow_new_txns(); + static void increment_txns(int); mdb_threadinfo* m_tinfo; MDB_txn* m_txn; diff --git a/src/common/download.cpp b/src/common/download.cpp index 3dd9c3976..c3dfa43d5 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -53,7 +53,7 @@ namespace tools download_thread_control(const std::string &path, const std::string &uri, std::function<void(const std::string&, const std::string&, bool)> result_cb, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb): path(path), uri(uri), result_cb(result_cb), progress_cb(progress_cb), stop(false), stopped(false), success(false) {} - ~download_thread_control() { if (thread.joinable()) thread.detach(); } + ~download_thread_control() { if (thread.joinable()) { thread.detach(); thread = {}; } } }; static void download_thread(download_async_handle control) @@ -293,9 +293,13 @@ namespace tools { boost::lock_guard<boost::mutex> lock(control->mutex); if (control->stopped) + { + control->thread = {}; return true; + } } control->thread.join(); + control->thread = {}; return true; } @@ -305,10 +309,14 @@ namespace tools { boost::lock_guard<boost::mutex> lock(control->mutex); if (control->stopped) + { + control->thread = {}; return true; + } control->stop = true; } control->thread.join(); + control->thread = {}; return true; } } diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 5a773f3cf..38aeeee54 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -51,6 +51,12 @@ #define INIT_SIZE_BLK 8 #define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) +#if defined(_MSC_VER) +#define THREADV __declspec(thread) +#else +#define THREADV __thread +#endif + extern void aesb_single_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); @@ -89,6 +95,28 @@ static inline int use_v4_jit(void) #endif } +#if defined(__x86_64__) || defined(__aarch64__) +static inline int force_software_aes(void) +{ + static int use = -1; + + if (use != -1) + return use; + + const char *env = getenv("MONERO_USE_SOFTWARE_AES"); + if (!env) { + use = 0; + } + else if (!strcmp(env, "0") || !strcmp(env, "no")) { + use = 0; + } + else { + use = 1; + } + return use; +} +#endif + #define VARIANT1_1(p) \ do if (variant == 1) \ { \ @@ -437,12 +465,6 @@ static inline int use_v4_jit(void) _b1 = _b; \ _b = _c; \ -#if defined(_MSC_VER) -#define THREADV __declspec(thread) -#else -#define THREADV __thread -#endif - #pragma pack(push, 1) union cn_slow_hash_state { @@ -498,25 +520,6 @@ STATIC INLINE void xor64(uint64_t *a, const uint64_t b) * @return true if the CPU supports AES, false otherwise */ -STATIC INLINE int force_software_aes(void) -{ - static int use = -1; - - if (use != -1) - return use; - - const char *env = getenv("MONERO_USE_SOFTWARE_AES"); - if (!env) { - use = 0; - } - else if (!strcmp(env, "0") || !strcmp(env, "no")) { - use = 0; - } - else { - use = 1; - } - return use; -} STATIC INLINE int check_aes_hw(void) { @@ -1009,6 +1012,44 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } #elif !defined NO_AES && (defined(__arm__) || defined(__aarch64__)) +#ifdef __aarch64__ +#include <sys/mman.h> +THREADV uint8_t *hp_state = NULL; +THREADV int hp_malloced = 0; + +void cn_slow_hash_allocate_state(void) +{ + if(hp_state != NULL) + return; + +#ifndef MAP_HUGETLB +#define MAP_HUGETLB 0 +#endif + hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_HUGETLB, -1, 0); + + if(hp_state == MAP_FAILED) + hp_state = NULL; + if(hp_state == NULL) + { + hp_malloced = 1; + hp_state = (uint8_t *) malloc(MEMORY); + } +} + +void cn_slow_hash_free_state(void) +{ + if(hp_state == NULL) + return; + + if (hp_malloced) + free(hp_state); + else + munmap(hp_state, MEMORY); + hp_state = NULL; + hp_malloced = 0; +} +#else void cn_slow_hash_allocate_state(void) { // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c @@ -1020,6 +1061,7 @@ void cn_slow_hash_free_state(void) // As above return; } +#endif #if defined(__GNUC__) #define RDATA_ALIGN16 __attribute__ ((aligned(16))) @@ -1060,6 +1102,23 @@ union cn_slow_hash_state * and moving between vector and regular registers stalls the pipeline. */ #include <arm_neon.h> +#ifndef __APPLE__ +#include <sys/auxv.h> +#include <asm/hwcap.h> +#endif + +STATIC INLINE int check_aes_hw(void) +{ +#ifdef __APPLE__ + return 1; +#else + static int supported = -1; + + if(supported < 0) + supported = (getauxval(AT_HWCAP) & HWCAP_AES) != 0; + return supported; +#endif +} #define TOTALBLOCKS (MEMORY / AES_BLOCK_SIZE) @@ -1156,7 +1215,6 @@ __asm__( STATIC INLINE void aes_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey, int nblocks) { const uint8x16_t *k = (const uint8x16_t *)expandedKey, zero = {0}; - uint8x16_t tmp; int i; for (i=0; i<nblocks; i++) @@ -1191,7 +1249,6 @@ STATIC INLINE void aes_pseudo_round_xor(const uint8_t *in, uint8_t *out, const u { const uint8x16_t *k = (const uint8x16_t *)expandedKey; const uint8x16_t *x = (const uint8x16_t *)xor; - uint8x16_t tmp; int i; for (i=0; i<nblocks; i++) @@ -1244,16 +1301,17 @@ STATIC INLINE void aligned_free(void *ptr) } #endif /* FORCE_USE_HEAP */ +STATIC INLINE void xor_blocks(uint8_t* a, const uint8_t* b) +{ + U64(a)[0] ^= U64(b)[0]; + U64(a)[1] ^= U64(b)[1]; +} + void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { RDATA_ALIGN16 uint8_t expandedKey[240]; -#ifndef FORCE_USE_HEAP - RDATA_ALIGN16 uint8_t local_hp_state[MEMORY]; -#else - uint8_t *local_hp_state = (uint8_t *)aligned_malloc(MEMORY,16); -#endif - + uint8_t *local_hp_state; uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; RDATA_ALIGN16 uint64_t b[4]; @@ -1264,12 +1322,22 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int size_t i, j; uint64_t *p = NULL; + oaes_ctx *aes_ctx = NULL; + int useAes = !force_software_aes() && check_aes_hw(); static void (*const extra_hashes[4])(const void *, size_t, char *) = { hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein }; + // this isn't supposed to happen, but guard against it for now. + if(hp_state == NULL) + cn_slow_hash_allocate_state(); + + // locals to avoid constant TLS dereferencing + local_hp_state = hp_state; + + // locals to avoid constant TLS dereferencing /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { @@ -1287,11 +1355,26 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int * the 2MB large random access buffer. */ - aes_expand_key(state.hs.b, expandedKey); - for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) + if(useAes) { - aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + aes_expand_key(state.hs.b, expandedKey); + for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) + { + aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + } + } + else + { + aes_ctx = (oaes_ctx *) oaes_alloc(); + oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE); + for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) + { + for(j = 0; j < INIT_SIZE_BLK; j++) + aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); + + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + } } U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0]; @@ -1307,13 +1390,26 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int _b = vld1q_u8((const uint8_t *)b); _b1 = vld1q_u8(((const uint8_t *)b) + AES_BLOCK_SIZE); - for(i = 0; i < ITER / 2; i++) + if(useAes) { - pre_aes(); - _c = vaeseq_u8(_c, zero); - _c = vaesmcq_u8(_c); - _c = veorq_u8(_c, _a); - post_aes(); + for(i = 0; i < ITER / 2; i++) + { + pre_aes(); + _c = vaeseq_u8(_c, zero); + _c = vaesmcq_u8(_c); + _c = veorq_u8(_c, _a); + post_aes(); + } + } + else + { + for(i = 0; i < ITER / 2; i++) + { + pre_aes(); + aesb_single_round((uint8_t *) &_c, (uint8_t *) &_c, (uint8_t *) &_a); + post_aes(); + } + } /* CryptoNight Step 4: Sequentially pass through the mixing buffer and use 10 rounds @@ -1322,11 +1418,27 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); - aes_expand_key(&state.hs.b[32], expandedKey); - for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) + if(useAes) + { + aes_expand_key(&state.hs.b[32], expandedKey); + for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) + { + // add the xor to the pseudo round + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + } + } + else { - // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + oaes_key_import_data(aes_ctx, &state.hs.b[32], AES_KEY_SIZE); + for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) + { + for(j = 0; j < INIT_SIZE_BLK; j++) + { + xor_blocks(&text[j * AES_BLOCK_SIZE], &local_hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); + } + } + oaes_free((OAES_CTX **) &aes_ctx); } /* CryptoNight Step 5: Apply Keccak to the state again, and then @@ -1339,10 +1451,6 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(state.init, text, INIT_SIZE_BYTE); hash_permutation(&state.hs); extra_hashes[state.hs.b[0] & 3](&state, 200, hash); - -#ifdef FORCE_USE_HEAP - aligned_free(local_hp_state); -#endif } #else /* aarch64 && crypto */ diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 6394a7071..d111f6f32 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -49,6 +49,7 @@ #include "misc_language.h" #include "ringct/rctTypes.h" #include "device/device.hpp" +#include "cryptonote_basic/fwd.h" namespace cryptonote { diff --git a/src/cryptonote_basic/events.h b/src/cryptonote_basic/events.h index 6c6742215..3417ece8c 100644 --- a/src/cryptonote_basic/events.h +++ b/src/cryptonote_basic/events.h @@ -41,6 +41,8 @@ namespace cryptonote { cryptonote::transaction tx; crypto::hash hash; + uint64_t blob_size; + uint64_t weight; bool res; //!< Listeners must ignore `tx` when this is false. }; } diff --git a/src/cryptonote_basic/fwd.h b/src/cryptonote_basic/fwd.h index d54223461..901ad151b 100644 --- a/src/cryptonote_basic/fwd.h +++ b/src/cryptonote_basic/fwd.h @@ -33,4 +33,5 @@ namespace cryptonote struct block; class transaction; struct txpool_event; + struct tx_block_template_backlog_entry; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index f0e6794b9..34031fb7c 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -588,6 +588,7 @@ block Blockchain::pop_block_from_blockchain() CHECK_AND_ASSERT_THROW_MES(m_db->height() > 1, "Cannot pop the genesis block"); + const uint8_t previous_hf_version = get_current_hard_fork_version(); try { m_db->pop_block(popped_block, popped_txs); @@ -650,6 +651,13 @@ block Blockchain::pop_block_from_blockchain() m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); invalidate_block_template_cache(); + const uint8_t new_hf_version = get_current_hard_fork_version(); + if (new_hf_version != previous_hf_version) + { + MINFO("Validating txpool for v" << (unsigned)new_hf_version); + m_tx_pool.validate(new_hf_version); + } + return popped_block; } //------------------------------------------------------------------ @@ -1238,6 +1246,12 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + crypto::hash prev_id; + if (!get_block_hash(alt_chain.back().bl, prev_id)) + MERROR("Failed to get block hash of an alternative chain's tip"); + else + send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + for (const auto& notifier : m_block_notifiers) { std::size_t notify_height = split_height; @@ -1784,6 +1798,30 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //------------------------------------------------------------------ +bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + prev_id = m_db->top_block_hash(&height); + ++height; + + major_version = m_hardfork->get_ideal_version(height); + + seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + difficulty = get_difficulty_for_next_block(); + median_weight = m_current_block_cumul_weight_median; + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + + m_tx_pool.get_block_template_backlog(tx_backlog); + + return true; +} +//------------------------------------------------------------------ // for an alternate chain, get the timestamps from the main chain to complete // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const @@ -4362,6 +4400,20 @@ leave: get_difficulty_for_next_block(); // just to cache it invalidate_block_template_cache(); + const uint8_t new_hf_version = get_current_hard_fork_version(); + if (new_hf_version != hf_version) + { + // the genesis block is added before everything's setup, and the txpool is empty + // when we start from scratch, so we skip this + const bool is_genesis_block = new_height == 1; + if (!is_genesis_block) + { + MGINFO("Validating txpool for v" << (unsigned)new_hf_version); + m_tx_pool.validate(new_hf_version); + } + } + + send_miner_notifications(id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); @@ -4990,6 +5042,8 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete unsigned nblocks = batches; if (i < extra) ++nblocks; + if (nblocks == 0) + break; tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, epee::span<const block>(&blocks[thread_height - height], nblocks), std::ref(maps[i])), true); thread_height += nblocks; } @@ -5270,7 +5324,7 @@ void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint m_max_prepare_blocks_threads = maxthreads; } -void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)>&& notify) +void Blockchain::add_block_notify(BlockNotifyCallback&& notify) { if (notify) { @@ -5279,6 +5333,15 @@ void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span } } +void Blockchain::add_miner_notify(MinerNotifyCallback&& notify) +{ + if (notify) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_miner_notifiers.push_back(std::move(notify)); + } +} + void Blockchain::safesyncmode(const bool onoff) { /* all of this is no-op'd if the user set a specific @@ -5531,6 +5594,33 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_valid = true; } +void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins) +{ + if (m_miner_notifiers.empty()) + return; + + const uint64_t height = m_db->height(); + const uint8_t major_version = m_hardfork->get_ideal_version(height); + const difficulty_type diff = get_difficulty_for_next_block(); + const uint64_t median_weight = m_current_block_cumul_weight_median; + + crypto::hash seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + std::vector<tx_block_template_backlog_entry> tx_backlog; + m_tx_pool.get_block_template_backlog(tx_backlog); + + for (const auto& notifier : m_miner_notifiers) + { + notifier(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + } +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index a0e7967de..9afbfbc2d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -90,6 +90,9 @@ namespace cryptonote */ typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback; + typedef boost::function<void(uint64_t /* height */, epee::span<const block> /* blocks */)> BlockNotifyCallback; + typedef boost::function<void(uint8_t /* major_version */, uint64_t /* height */, const crypto::hash& /* prev_id */, const crypto::hash& /* seed_hash */, difficulty_type /* diff */, uint64_t /* median_weight */, uint64_t /* already_generated_coins */, const std::vector<tx_block_template_backlog_entry>& /* tx_backlog */)> MinerNotifyCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -371,6 +374,22 @@ namespace cryptonote bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @brief gets data required to create a block template and start mining on it + * + * @param major_version current hardfork version + * @param height current blockchain height + * @param prev_id hash of the top block + * @param seed_hash seed hash used for RandomX initialization + * @param difficulty current mining difficulty + * @param median_weight current median block weight + * @param already_generated_coins current emission + * @param tx_backlog transactions in mempool ready to be mined + * + * @return true if block template filled in successfully, else false + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief checks if a block is known about with a given hash * * This function checks the main chain, alternate chains, and invalid blocks @@ -775,7 +794,14 @@ namespace cryptonote * * @param notify the notify object to call at every new block */ - void add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)> &¬ify); + void add_block_notify(BlockNotifyCallback&& notify); + + /** + * @brief sets a miner notify object to call for every new block + * + * @param notify the notify object to call at every new block + */ + void add_miner_notify(MinerNotifyCallback&& notify); /** * @brief sets a reorg notify object to call for every reorg @@ -1157,7 +1183,8 @@ namespace cryptonote the callable object has a single `std::shared_ptr` or `std::weap_ptr` internally. Whereas, the libstdc++ `std::function` will allocate. */ - std::vector<boost::function<void(std::uint64_t, epee::span<const block>)>> m_block_notifiers; + std::vector<BlockNotifyCallback> m_block_notifiers; + std::vector<MinerNotifyCallback> m_miner_notifiers; std::shared_ptr<tools::Notify> m_reorg_notify; // for prepare_handle_incoming_blocks @@ -1537,5 +1564,13 @@ namespace cryptonote * At some point, may be used to push an update to miners */ void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t seed_height, const crypto::hash &seed_hash, uint64_t pool_cookie); + + /** + * @brief sends new block notifications to ZMQ `miner_data` subscribers + * + * @param prev_id hash of new blockchain tip + * @param already_generated_coins total coins mined by the network so far + */ + void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index ed9f7a28c..4c6536318 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1065,8 +1065,9 @@ namespace cryptonote if (already_have[i]) continue; - const uint64_t weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], tx_relay, relayed); + results[i].blob_size = it->blob.size(); + results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, results[i].weight, tvc[i], tx_relay, relayed); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} @@ -1405,6 +1406,11 @@ namespace cryptonote return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //----------------------------------------------------------------------------------------------- + bool core::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) + { + return m_blockchain_storage.get_miner_data(major_version, height, prev_id, seed_hash, difficulty, median_weight, already_generated_coins, tx_backlog); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 286145031..d2bffdaee 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -237,6 +237,13 @@ namespace cryptonote virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @copydoc Blockchain::get_miner_data + * + * @note see Blockchain::get_miner_data + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief called when a transaction is relayed. * @note Should only be invoked from `levin_notify`. */ diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index f41c63a4b..f6061b803 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -678,7 +678,7 @@ namespace cryptonote rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1); } - bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners) + bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners) { // block 202612 bug workaround if (height == 202612) @@ -687,8 +687,7 @@ namespace cryptonote epee::string_tools::hex_to_pod(longhash_202612, res); return true; } - blobdata bd = get_block_hashing_blob(b); - if (b.major_version >= RX_BLOCK_VERSION) + if (major_version >= RX_BLOCK_VERSION) { uint64_t seed_height, main_height; crypto::hash hash; @@ -705,12 +704,18 @@ namespace cryptonote } rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash); } else { - const int pow_variant = b.major_version >= 7 ? b.major_version - 6 : 0; + const int pow_variant = major_version >= 7 ? major_version - 6 : 0; crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height); } return true; } + bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners) + { + blobdata bd = get_block_hashing_blob(b); + return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners); + } + bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners) { return get_block_longhash(pbc, b, res, height, NULL, miners); diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 73cdd31cd..cea4aad17 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -108,6 +108,15 @@ namespace cryptonote }; //--------------------------------------------------------------- + + struct tx_block_template_backlog_entry + { + crypto::hash id; + uint64_t weight; + uint64_t fee; + }; + + //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); @@ -133,6 +142,8 @@ namespace cryptonote ); class Blockchain; + bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, + const int major_version, const crypto::hash *seed_hash, const int miners); bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners); bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners); void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index a7e96e23a..6fe5a54ac 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -913,6 +913,32 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + void tx_memory_pool::get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); + txpool_tx_meta_t tmp_meta; + m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ + transaction tx; + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) + { + MERROR("Failed to parse tx from txpool"); + // continue + return true; + } + tx.set_hash(txid); + + tmp_meta = meta; + + if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx)) + backlog.push_back({txid, meta.weight, meta.fee}); + + return true; + }, true, category); + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1222,11 +1248,11 @@ namespace cryptonote return ret; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref& txblob, transaction &tx) const { - struct transction_parser + struct transaction_parser { - transction_parser(const cryptonote::blobdata &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} + transaction_parser(const cryptonote::blobdata_ref &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} cryptonote::transaction &operator()() { if (!parsed) @@ -1238,7 +1264,7 @@ namespace cryptonote } return tx; } - const cryptonote::blobdata &txblob; + const cryptonote::blobdata_ref &txblob; const crypto::hash &txid; transaction &tx; bool parsed; @@ -1289,6 +1315,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata& txblob, transaction &tx) const + { + return is_transaction_ready_to_go(txd, txid, cryptonote::blobdata_ref{txblob.data(), txblob.size()}, tx); + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::have_key_images(const std::unordered_set<crypto::key_image>& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) @@ -1537,61 +1568,59 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - size_t tx_weight_limit = get_transaction_weight_limit(version); - std::unordered_set<crypto::hash> remove; - m_txpool_weight = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*) { - m_txpool_weight += meta.weight; - if (meta.weight > tx_weight_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); - remove.insert(txid); - } - else if (m_blockchain.have_tx(txid)) { - LOG_PRINT_L1("Transaction " << txid << " is in the blockchain, removing it from pool"); - remove.insert(txid); - } + MINFO("Validating txpool contents for v" << (unsigned)version); + + LockedTXN lock(m_blockchain.get_db()); + + struct tx_entry_t + { + crypto::hash txid; + txpool_tx_meta_t meta; + }; + + // get all txids + std::vector<tx_entry_t> txes; + m_blockchain.for_all_txpool_txes([&txes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*) { + if (!meta.pruned) // skip pruned txes + txes.push_back({txid, meta}); return true; }, false, relay_category::all); - size_t n_removed = 0; - if (!remove.empty()) + // take them all out and add them back in, some might fail + size_t added = 0; + for (auto &e: txes) { - LockedTXN lock(m_blockchain.get_db()); - for (const crypto::hash &txid: remove) + try { - try - { - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) // remove pruned ones on startup, they're meant to be temporary - { - MERROR("Failed to parse tx from txpool"); - continue; - } - // remove tx from db first - m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= get_transaction_weight(tx, txblob.size()); - remove_transaction_keyimages(tx, txid); - auto sorted_it = find_tx_in_sorted_container(txid); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - { - LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); - } - else - { - m_txs_by_fee_and_receive_time.erase(sorted_it); - } - ++n_removed; - } - catch (const std::exception &e) + size_t weight; + uint64_t fee; + cryptonote::transaction tx; + cryptonote::blobdata blob; + bool relayed, do_not_relay, double_spend_seen, pruned; + if (!take_tx(e.txid, tx, blob, weight, fee, relayed, do_not_relay, double_spend_seen, pruned)) + MERROR("Failed to get tx " << e.txid << " from txpool for re-validation"); + + cryptonote::tx_verification_context tvc{}; + relay_method tx_relay = e.meta.get_relay_method(); + if (!add_tx(tx, e.txid, blob, e.meta.weight, tvc, tx_relay, relayed, version)) { - MERROR("Failed to remove invalid tx from pool"); - // continue + MINFO("Failed to re-validate tx " << e.txid << " for v" << (unsigned)version << ", dropped"); + continue; } + m_blockchain.update_txpool_tx(e.txid, e.meta); + ++added; + } + catch (const std::exception &e) + { + MERROR("Failed to re-validate tx from pool"); + continue; } - lock.commit(); } + + lock.commit(); + + const size_t n_removed = txes.size() - added; if (n_removed > 0) ++m_cookie; return n_removed; diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index ab2a57ea2..80b38c51d 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -266,6 +266,15 @@ namespace cryptonote void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive = false) const; /** + * @brief get (hash, weight, fee) for all transactions in the pool - the minimum required information to create a block template + * + * @param backlog return-by-reference that data + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes + * + */ + void get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive = false) const; + + /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics @@ -540,6 +549,7 @@ namespace cryptonote * * @return true if the transaction is good to go, otherwise false */ + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref &txblob, transaction&tx) const; bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction&tx) const; /** diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 106253082..39d562fd1 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -153,6 +153,7 @@ namespace cryptonote context.m_last_request_time = boost::date_time::not_a_date_time; context.m_expect_response = 0; context.m_expect_height = 0; + context.m_requested_objects.clear(); context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download } else diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 0b065c3c3..53de407b6 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -287,6 +287,12 @@ namespace levin boost::asio::steady_timer next_epoch; boost::asio::steady_timer flush_txs; boost::asio::io_service::strand strand; + struct context_t { + std::vector<cryptonote::blobdata> fluff_txs; + std::chrono::steady_clock::time_point flush_time; + bool m_is_income; + }; + boost::unordered_map<boost::uuids::uuid, context_t> contexts; net::dandelionpp::connection_map map;//!< Tracks outgoing uuid's for noise channels or Dandelion++ stems std::deque<noise_channel> channels; //!< Never touch after init; only update elements on `noise_channel.strand` std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time @@ -363,14 +369,16 @@ namespace levin const auto now = std::chrono::steady_clock::now(); auto next_flush = std::chrono::steady_clock::time_point::max(); std::vector<std::pair<std::vector<blobdata>, boost::uuids::uuid>> connections{}; - zone_->p2p->foreach_connection([timer_error, now, &next_flush, &connections] (detail::p2p_context& context) + for (auto &e: zone_->contexts) { + auto &id = e.first; + auto &context = e.second; if (!context.fluff_txs.empty()) { if (context.flush_time <= now || timer_error) // flush on canceled timer { context.flush_time = std::chrono::steady_clock::time_point::max(); - connections.emplace_back(std::move(context.fluff_txs), context.m_connection_id); + connections.emplace_back(std::move(context.fluff_txs), id); context.fluff_txs.clear(); } else // not flushing yet @@ -378,8 +386,7 @@ namespace levin } else // nothing to flush context.flush_time = std::chrono::steady_clock::time_point::max(); - return true; - }); + } /* Always send with `fluff` flag, even over i2p/tor. The hidden service will disable the forwarding delay and immediately fluff. The i2p/tor @@ -427,22 +434,21 @@ namespace levin MDEBUG("Queueing " << txs.size() << " transaction(s) for Dandelion++ fluffing"); - - zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context) + for (auto &e: zone->contexts) { + auto &id = e.first; + auto &context = e.second; // When i2p/tor, only fluff to outbound connections - if (context.handshake_complete() && source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) + if (source != id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) { if (context.fluff_txs.empty()) context.flush_time = now + (context.m_is_income ? in_duration() : out_duration()); next_flush = std::min(next_flush, context.flush_time); context.fluff_txs.reserve(context.fluff_txs.size() + txs.size()); - for (const blobdata& tx : txs) - context.fluff_txs.push_back(tx); // must copy instead of move (multiple conns) + context.fluff_txs.insert(context.fluff_txs.end(), txs.begin(), txs.end()); } - return true; - }); + } if (next_flush == std::chrono::steady_clock::time_point::max()) MWARNING("Unable to send transaction(s), no available connections"); @@ -749,6 +755,32 @@ namespace levin ); } + void notify::on_handshake_complete(const boost::uuids::uuid &id, bool is_income) + { + if (!zone_) + return; + + auto& zone = zone_; + zone_->strand.dispatch([zone, id, is_income]{ + zone->contexts[id] = { + .fluff_txs = {}, + .flush_time = std::chrono::steady_clock::time_point::max(), + .m_is_income = is_income, + }; + }); + } + + void notify::on_connection_close(const boost::uuids::uuid &id) + { + if (!zone_) + return; + + auto& zone = zone_; + zone_->strand.dispatch([zone, id]{ + zone->contexts.erase(id); + }); + } + void notify::run_epoch() { if (!zone_) diff --git a/src/cryptonote_protocol/levin_notify.h b/src/cryptonote_protocol/levin_notify.h index abbf9d461..12704746a 100644 --- a/src/cryptonote_protocol/levin_notify.h +++ b/src/cryptonote_protocol/levin_notify.h @@ -101,6 +101,9 @@ namespace levin //! Probe for new outbound connection - skips if not needed. void new_out_connection(); + void on_handshake_complete(const boost::uuids::uuid &id, bool is_income); + void on_connection_close(const boost::uuids::uuid &id); + //! Run the logic for the next epoch immediately. Only use in testing. void run_epoch(); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 99430b2b0..3f1885423 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -120,6 +120,7 @@ public: if (shared) { core.get().get_blockchain_storage().add_block_notify(cryptonote::listener::zmq_pub::chain_main{shared}); + core.get().get_blockchain_storage().add_miner_notify(cryptonote::listener::zmq_pub::miner_data{shared}); core.get().set_txpool_listener(cryptonote::listener::zmq_pub::txpool_add{shared}); } } diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index d435b448c..07009b9d2 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -162,6 +162,26 @@ namespace hw { * Live refresh process termination */ virtual void live_refresh_finish() =0; + + /** + * Requests public address, uses empty passphrase if asked for. + */ + virtual bool get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) =0; + + /** + * Reset session ID, restart with a new session. + */ + virtual void reset_session() =0; + + /** + * Returns true if device already asked for passphrase entry before (i.e., obviously supports passphrase entry) + */ + virtual bool seen_passphrase_entry_prompt() =0; + + /** + * Uses empty passphrase for all passphrase queries. + */ + virtual void set_use_empty_passphrase(bool always_use_empty_passphrase) =0; }; } diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 5caad3a1a..ebad740cd 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -451,13 +451,6 @@ namespace hw { ASSERT_X(this->length_recv>=3, "Communication error, less than three bytes received. Check your application version."); - unsigned int device_version = 0; - device_version = VERSION(this->buffer_recv[0], this->buffer_recv[1], this->buffer_recv[2]); - - ASSERT_X (device_version >= MINIMAL_APP_VERSION, - "Unsupported device application version: " << VERSION_MAJOR(device_version)<<"."<<VERSION_MINOR(device_version)<<"."<<VERSION_MICRO(device_version) << - " At least " << MINIMAL_APP_VERSION_MAJOR<<"."<<MINIMAL_APP_VERSION_MINOR<<"."<<MINIMAL_APP_VERSION_MICRO<<" is required."); - return true; } @@ -470,6 +463,9 @@ namespace hw { this->length_recv -= 2; this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; logRESP(); + MDEBUG("Device "<< this->id << " exchange: sw: " << this->sw << " expected: " << ok); + ASSERT_X(sw != SW_CLIENT_NOT_SUPPORTED, "Monero Ledger App doesn't support current monero version. Try to update the Monero Ledger App, at least " << MINIMAL_APP_VERSION_MAJOR<< "." << MINIMAL_APP_VERSION_MINOR << "." << MINIMAL_APP_VERSION_MICRO << " is required."); + ASSERT_X(sw != SW_PROTOCOL_NOT_SUPPORTED, "Make sure no other program is communicating with the Ledger."); ASSERT_SW(this->sw,ok,msk); return this->sw; diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index 3b6cc505f..590ae41b5 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -166,8 +166,6 @@ namespace hw { void send_secret(const unsigned char sec[32], int &offset); void receive_secret(unsigned char sec[32], int &offset); - // hw running mode - device_mode mode; bool tx_in_progress; // map public destination key to ephemeral destination key diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index c2070b0d1..0545f3f26 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -66,8 +66,8 @@ namespace trezor { device_trezor::~device_trezor() { try { - disconnect(); - release(); + device_trezor::disconnect(); + device_trezor::release(); } catch(std::exception const& e){ MWARNING("Could not disconnect and release: " << e.what()); } @@ -178,6 +178,15 @@ namespace trezor { } } + bool device_trezor::get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) { + m_reply_with_empty_passphrase = true; + const auto empty_passphrase_reverter = epee::misc_utils::create_scope_leave_handler([&]() { + m_reply_with_empty_passphrase = false; + }); + + return get_public_address(pubkey); + } + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { try { MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); @@ -206,6 +215,18 @@ namespace trezor { get_address(index, payment_id, true); } + void device_trezor::reset_session() { + m_device_session_id.clear(); + } + + bool device_trezor::seen_passphrase_entry_prompt() { + return m_seen_passphrase_entry_message; + } + + void device_trezor::set_use_empty_passphrase(bool always_use_empty_passphrase) { + m_always_use_empty_passphrase = always_use_empty_passphrase; + } + /* ======================================================================= */ /* Helpers */ /* ======================================================================= */ diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index d91d1de3f..15337d2b4 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -205,6 +205,26 @@ namespace trezor { const ::tools::wallet2::unsigned_tx_set & unsigned_tx, ::tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) override; + + /** + * Requests public address, uses empty passphrase if asked for. + */ + bool get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) override; + + /** + * Reset session ID, restart with a new session. + */ + virtual void reset_session() override; + + /** + * Returns true if device already asked for passphrase entry before (i.e., obviously supports passphrase entry) + */ + bool seen_passphrase_entry_prompt() override; + + /** + * Uses empty passphrase for all passphrase queries. + */ + void set_use_empty_passphrase(bool use_always_empty_passphrase) override; }; #endif diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index b0b4342f5..016eb2816 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -45,7 +45,10 @@ namespace trezor { const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; - device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success) { + device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success), + m_reply_with_empty_passphrase(false), + m_always_use_empty_passphrase(false), + m_seen_passphrase_entry_message(false) { #ifdef WITH_TREZOR_DEBUGGING m_debug = false; #endif @@ -155,6 +158,9 @@ namespace trezor { TREZOR_AUTO_LOCK_DEVICE(); m_device_session_id.clear(); m_features.reset(); + m_seen_passphrase_entry_message = false; + m_reply_with_empty_passphrase = false; + m_always_use_empty_passphrase = false; if (m_transport){ try { @@ -476,6 +482,7 @@ namespace trezor { return; } + m_seen_passphrase_entry_message = true; bool on_device = true; if (msg->has__on_device() && !msg->_on_device()){ on_device = false; // do not enter on device, old devices. @@ -491,19 +498,21 @@ namespace trezor { } boost::optional<epee::wipeable_string> passphrase; - TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + if (m_reply_with_empty_passphrase || m_always_use_empty_passphrase) { + MDEBUG("Answering passphrase prompt with an empty passphrase, always use empty: " << m_always_use_empty_passphrase); + on_device = false; + passphrase = epee::wipeable_string(""); + } else if (m_passphrase){ + MWARNING("Answering passphrase prompt with a stored passphrase (do not use; passphrase can be seen by a potential malware / attacker)"); + on_device = false; + passphrase = epee::wipeable_string(m_passphrase.get()); + } else { + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + } messages::common::PassphraseAck m; m.set_on_device(on_device); if (!on_device) { - if (!passphrase && m_passphrase) { - passphrase = m_passphrase; - } - - if (m_passphrase) { - m_passphrase = boost::none; - } - if (passphrase) { m.set_allocated_passphrase(new std::string(passphrase->data(), passphrase->size())); } diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 0162b23df..de49397d5 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -101,6 +101,9 @@ namespace trezor { messages::MessageType m_last_msg_type; cryptonote::network_type network_type; + bool m_reply_with_empty_passphrase; + bool m_always_use_empty_passphrase; + bool m_seen_passphrase_entry_message; #ifdef WITH_TREZOR_DEBUGGING std::shared_ptr<trezor_debug_callback> m_debug_callback; diff --git a/src/net/socks.cpp b/src/net/socks.cpp index c2330bd41..6463e669e 100644 --- a/src/net/socks.cpp +++ b/src/net/socks.cpp @@ -321,8 +321,9 @@ namespace socks { if (self && self->proxy_.is_open()) { - self->proxy_.shutdown(boost::asio::ip::tcp::socket::shutdown_both); - self->proxy_.close(); + boost::system::error_code ec; + self->proxy_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); + self->proxy_.close(ec); } }); } diff --git a/src/net/socks.h b/src/net/socks.h index 739c972ab..506b53195 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -201,6 +201,13 @@ namespace socks std::shared_ptr<client> self_; void operator()(boost::system::error_code error = boost::system::error_code{}); }; + + //! Calls `async_close` on `self` at destruction. NOP if `nullptr`. + struct close_on_exit + { + std::shared_ptr<client> self; + ~close_on_exit() { async_close{std::move(self)}(); } + }; }; template<typename Handler> diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index c951db085..d9050200a 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -342,6 +342,7 @@ namespace nodetool } }; + net::socks::client::close_on_exit close_client{}; boost::unique_future<client_result> socks_result{}; { boost::promise<client_result> socks_promise{}; @@ -350,6 +351,7 @@ namespace nodetool auto client = net::socks::make_connect_client( boost::asio::ip::tcp::socket{service}, net::socks::version::v4a, notify{std::move(socks_promise)} ); + close_client.self = client; if (!start_socks(std::move(client), proxy, remote)) return boost::none; } @@ -371,7 +373,10 @@ namespace nodetool { auto result = socks_result.get(); if (!result.first) + { + close_client.self.reset(); return {std::move(result.second)}; + } MERROR("Failed to make socks connection to " << remote.str() << " (via " << proxy << "): " << result.first.message()); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 9e64121be..ac815a100 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -111,15 +111,11 @@ namespace nodetool struct p2p_connection_context_t: base_type //t_payload_net_handler::connection_context //public net_utils::connection_context_base { p2p_connection_context_t() - : fluff_txs(), - flush_time(std::chrono::steady_clock::time_point::max()), - peer_id(0), + : peer_id(0), support_flags(0), m_in_timedsync(false) {} - std::vector<cryptonote::blobdata> fluff_txs; - std::chrono::steady_clock::time_point flush_time; peerid_type peer_id; uint32_t support_flags; bool m_in_timedsync; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index ac65a57c1..d4b39869c 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1429,6 +1429,7 @@ namespace nodetool ape.first_seen = first_seen_stamp ? first_seen_stamp : time(nullptr); zone.m_peerlist.append_with_peer_anchor(ape); + zone.m_notifier.on_handshake_complete(con->m_connection_id, con->m_is_income); zone.m_notifier.new_out_connection(); LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK."); @@ -2543,6 +2544,8 @@ namespace nodetool return 1; } + zone.m_notifier.on_handshake_complete(context.m_connection_id, context.m_is_income); + if(has_too_many_connections(context.m_remote_address)) { LOG_PRINT_CCONTEXT_L1("CONNECTION FROM " << context.m_remote_address.host_str() << " REFUSED, too many connections from the same address"); @@ -2669,6 +2672,9 @@ namespace nodetool zone.m_peerlist.remove_from_peer_anchor(na); } + if (!zone.m_net_server.is_stop_signal_sent()) { + zone.m_notifier.on_connection_close(context.m_connection_id); + } m_payload_handler.on_connection_close(context); MINFO("["<< epee::net_utils::print_connection_context(context) << "] CLOSE CONNECTION"); diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 602ed822f..e114ea7c6 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -972,14 +972,26 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::vector<std::string>::const_iterator txhi = req.txs_hashes.begin(); - std::vector<crypto::hash>::const_iterator vhi = vh.begin(); + CHECK_AND_ASSERT_MES(txs.size() + missed_txs.size() == vh.size(), false, "mismatched number of txs"); + + auto txhi = req.txs_hashes.cbegin(); + auto vhi = vh.cbegin(); + auto missedi = missed_txs.cbegin(); + for(auto& tx: txs) { res.txs.push_back(COMMAND_RPC_GET_TRANSACTIONS::entry()); COMMAND_RPC_GET_TRANSACTIONS::entry &e = res.txs.back(); + while (missedi != missed_txs.end() && *missedi == *vhi) + { + ++vhi; + ++txhi; + ++missedi; + } + crypto::hash tx_hash = *vhi++; + CHECK_AND_ASSERT_MES(tx_hash == std::get<0>(tx), false, "mismatched tx hash"); e.tx_hash = *txhi++; e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); if (req.split || req.prune || std::get<3>(tx).empty()) @@ -1863,6 +1875,80 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + if(!check_core_ready()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy"; + return false; + } + + crypto::hash prev_id, seed_hash; + difficulty_type difficulty; + + std::vector<tx_block_template_backlog_entry> tx_backlog; + if (!m_core.get_miner_data(res.major_version, res.height, prev_id, seed_hash, difficulty, res.median_weight, res.already_generated_coins, tx_backlog)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: failed to get miner data"; + LOG_ERROR("Failed to get miner data"); + return false; + } + + res.tx_backlog.clear(); + res.tx_backlog.reserve(tx_backlog.size()); + + for (const auto& entry : tx_backlog) + { + res.tx_backlog.emplace_back(COMMAND_RPC_GETMINERDATA::response::tx_backlog_entry{string_tools::pod_to_hex(entry.id), entry.weight, entry.fee}); + } + + res.prev_id = string_tools::pod_to_hex(prev_id); + res.seed_hash = string_tools::pod_to_hex(seed_hash); + res.difficulty = cryptonote::hex(difficulty); + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_calcpow(const COMMAND_RPC_CALCPOW::request& req, COMMAND_RPC_CALCPOW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + RPC_TRACKER(calcpow); + + blobdata blockblob; + if(!string_tools::parse_hexstr_to_binbuff(req.block_blob, blockblob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + if(!m_core.check_incoming_block_size(blockblob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB_SIZE; + error_resp.message = "Block blob size is too big, rejecting block"; + return false; + } + crypto::hash seed_hash, pow_hash; + std::string buf; + if(req.seed_hash.size()) + { + if (!string_tools::parse_hexstr_to_binbuff(req.seed_hash, buf) || + buf.size() != sizeof(crypto::hash)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Wrong seed hash"; + return false; + } + buf.copy(reinterpret_cast<char *>(&seed_hash), sizeof(crypto::hash)); + } + + cryptonote::get_block_longhash(&(m_core.get_blockchain_storage()), blockblob, pow_hash, req.height, + req.major_version, req.seed_hash.size() ? &seed_hash : NULL, 0); + res = string_tools::pod_to_hex(pow_hash); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(add_aux_pow); @@ -3115,6 +3201,14 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + if (restricted && req.amounts != std::vector<uint64_t>(1, 0)) + { + error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED; + error_resp.message = "Restricted RPC can only get output distribution for rct outputs. Use your own node."; + return false; + } + size_t n_0 = 0, n_non0 = 0; for (uint64_t amount: req.amounts) if (amount) ++n_non0; else ++n_0; @@ -3156,6 +3250,13 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::BIN, "/get_output_distribution.bin", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + if (restricted && req.amounts != std::vector<uint64_t>(1, 0)) + { + res.status = "Restricted RPC can only get output distribution for rct outputs. Use your own node."; + return false; + } + size_t n_0 = 0, n_non0 = 0; for (uint64_t amount: req.amounts) if (amount) ++n_non0; else ++n_0; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index db1429ab1..664af3686 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -148,6 +148,8 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("get_miner_data", on_getminerdata, COMMAND_RPC_GETMINERDATA) + MAP_JON_RPC_WE_IF("calc_pow", on_calcpow, COMMAND_RPC_CALCPOW, !m_restricted) MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) @@ -229,6 +231,8 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_calcpow(const COMMAND_RPC_CALCPOW::request& req, COMMAND_RPC_CALCPOW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index ff8c98b98..166fb39ea 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 7 +#define CORE_RPC_VERSION_MINOR 9 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -940,6 +940,78 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_GETMINERDATA + { + struct request_t: public rpc_request_base + { + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + uint8_t major_version; + uint64_t height; + std::string prev_id; + std::string seed_hash; + std::string difficulty; + uint64_t median_weight; + uint64_t already_generated_coins; + + struct tx_backlog_entry + { + std::string id; + uint64_t weight; + uint64_t fee; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(weight) + KV_SERIALIZE(fee) + END_KV_SERIALIZE_MAP() + }; + + std::vector<tx_backlog_entry> tx_backlog; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(major_version) + KV_SERIALIZE(height) + KV_SERIALIZE(prev_id) + KV_SERIALIZE(seed_hash) + KV_SERIALIZE(difficulty) + KV_SERIALIZE(median_weight) + KV_SERIALIZE(already_generated_coins) + KV_SERIALIZE(tx_backlog) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_CALCPOW + { + struct request_t: public rpc_request_base + { + uint8_t major_version; + uint64_t height; + blobdata block_blob; + std::string seed_hash; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + KV_SERIALIZE(major_version) + KV_SERIALIZE(height) + KV_SERIALIZE(block_blob) + KV_SERIALIZE(seed_hash) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + typedef std::string response; + }; + struct COMMAND_RPC_ADD_AUX_POW { struct aux_pow_t diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp index eac530968..074b55207 100644 --- a/src/rpc/zmq_pub.cpp +++ b/src/rpc/zmq_pub.cpp @@ -48,6 +48,8 @@ #include "cryptonote_basic/events.h" #include "misc_log_ex.h" #include "serialization/json_object.h" +#include "ringct/rctTypes.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.zmq" @@ -57,6 +59,7 @@ namespace constexpr const char txpool_signal[] = "tx_signal"; using chain_writer = void(epee::byte_stream&, std::uint64_t, epee::span<const cryptonote::block>); + using miner_writer = void(epee::byte_stream&, uint8_t, uint64_t, const crypto::hash&, const crypto::hash&, cryptonote::difficulty_type, uint64_t, uint64_t, const std::vector<cryptonote::tx_block_template_backlog_entry>&); using txpool_writer = void(epee::byte_stream&, epee::span<const cryptonote::txpool_event>); template<typename F> @@ -116,13 +119,30 @@ namespace const epee::span<const cryptonote::block> blocks; }; + //! Object for miner data serialization + struct miner_data + { + uint8_t major_version; + uint64_t height; + const crypto::hash& prev_id; + const crypto::hash& seed_hash; + cryptonote::difficulty_type diff; + uint64_t median_weight; + uint64_t already_generated_coins; + const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog; + }; + //! Object for "minimal" tx serialization struct minimal_txpool { const cryptonote::transaction& tx; + crypto::hash hash; + uint64_t blob_size; + uint64_t weight; + uint64_t fee; }; - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain& self) { namespace adapt = boost::adaptors; @@ -143,19 +163,27 @@ namespace dest.EndObject(); } - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const miner_data& self) { - crypto::hash id{}; - std::size_t blob_size = 0; - if (!get_transaction_hash(self.tx, id, blob_size)) - { - MERROR("ZMQ/Pub failure: get_transaction_hash"); - return; - } + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, major_version, self.major_version); + INSERT_INTO_JSON_OBJECT(dest, height, self.height); + INSERT_INTO_JSON_OBJECT(dest, prev_id, self.prev_id); + INSERT_INTO_JSON_OBJECT(dest, seed_hash, self.seed_hash); + INSERT_INTO_JSON_OBJECT(dest, difficulty, cryptonote::hex(self.diff)); + INSERT_INTO_JSON_OBJECT(dest, median_weight, self.median_weight); + INSERT_INTO_JSON_OBJECT(dest, already_generated_coins, self.already_generated_coins); + INSERT_INTO_JSON_OBJECT(dest, tx_backlog, self.tx_backlog); + dest.EndObject(); + } + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool& self) + { dest.StartObject(); - INSERT_INTO_JSON_OBJECT(dest, id, id); - INSERT_INTO_JSON_OBJECT(dest, blob_size, blob_size); + INSERT_INTO_JSON_OBJECT(dest, id, self.hash); + INSERT_INTO_JSON_OBJECT(dest, blob_size, self.blob_size); + INSERT_INTO_JSON_OBJECT(dest, weight, self.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, self.fee); dest.EndObject(); } @@ -169,6 +197,11 @@ namespace json_pub(buf, minimal_chain{height, blocks}); } + void json_miner_data(epee::byte_stream& buf, uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, cryptonote::difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog) + { + json_pub(buf, miner_data{major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog}); + } + // boost::adaptors are in place "views" - no copy/move takes place // moving transactions (via sort, etc.), is expensive! @@ -187,7 +220,7 @@ namespace namespace adapt = boost::adaptors; const auto to_minimal_tx = [](const cryptonote::txpool_event& event) { - return minimal_txpool{event.tx}; + return minimal_txpool{event.tx, event.hash, event.blob_size, event.weight, cryptonote::get_tx_fee(event.tx)}; }; json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_minimal_tx))); } @@ -198,6 +231,11 @@ namespace {u8"json-minimal-chain_main", json_minimal_chain} }}; + constexpr const std::array<context<miner_writer>, 1> miner_contexts = + {{ + {u8"json-full-miner_data", json_miner_data}, + }}; + constexpr const std::array<context<txpool_writer>, 2> txpool_contexts = {{ {u8"json-full-txpool_add", json_full_txpool}, @@ -321,6 +359,7 @@ namespace cryptonote { namespace listener zmq_pub::zmq_pub(void* context) : relay_(), chain_subs_{{0}}, + miner_subs_{{0}}, txpool_subs_{{0}}, sync_() { @@ -328,6 +367,7 @@ zmq_pub::zmq_pub(void* context) throw std::logic_error{"ZMQ context cannot be NULL"}; verify_sorted(chain_contexts, "chain_contexts"); + verify_sorted(miner_contexts, "miner_contexts"); verify_sorted(txpool_contexts, "txpool_contexts"); relay_.reset(zmq_socket(context, ZMQ_PAIR)); @@ -348,22 +388,25 @@ bool zmq_pub::sub_request(boost::string_ref message) message.remove_prefix(1); const auto chain_range = get_range(chain_contexts, message); + const auto miner_range = get_range(miner_contexts, message); const auto txpool_range = get_range(txpool_contexts, message); - if (!chain_range.empty() || !txpool_range.empty()) + if (!chain_range.empty() || !miner_range.empty() || !txpool_range.empty()) { MDEBUG("Client " << (tag ? "subscribed" : "unsubscribed") << " to " << - chain_range.size() << " chain topic(s) and " << txpool_range.size() << " txpool topic(s)"); + chain_range.size() << " chain topic(s), " << miner_range.size() << " miner topic(s) and " << txpool_range.size() << " txpool topic(s)"); const boost::lock_guard<boost::mutex> lock{sync_}; switch (tag) { case 0: remove_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + remove_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); remove_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; case 1: add_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + add_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); add_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; default: @@ -436,6 +479,25 @@ std::size_t zmq_pub::send_chain_main(const std::uint64_t height, const epee::spa return 0; } +std::size_t zmq_pub::send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + boost::unique_lock<boost::mutex> guard{sync_}; + + const auto subs_copy = miner_subs_; + guard.unlock(); + + for (const std::size_t sub : subs_copy) + { + if (sub) + { + auto messages = make_pubs(subs_copy, miner_contexts, major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + guard.lock(); + return send_messages(relay_.get(), messages); + } + } + return 0; +} + std::size_t zmq_pub::send_txpool_add(std::vector<txpool_event> txes) { if (txes.empty()) @@ -466,6 +528,15 @@ void zmq_pub::chain_main::operator()(const std::uint64_t height, epee::span<cons MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); } +void zmq_pub::miner_data::operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const +{ + const std::shared_ptr<zmq_pub> self = self_.lock(); + if (self) + self->send_miner_data(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + else + MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); +} + void zmq_pub::txpool_add::operator()(std::vector<cryptonote::txpool_event> txes) const { const std::shared_ptr<zmq_pub> self = self_.lock(); diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h index 02e6b8103..c636e1d7b 100644 --- a/src/rpc/zmq_pub.h +++ b/src/rpc/zmq_pub.h @@ -39,6 +39,7 @@ #include "cryptonote_basic/fwd.h" #include "net/zmq.h" #include "span.h" +#include "cryptonote_basic/difficulty.h" namespace cryptonote { namespace listener { @@ -59,6 +60,7 @@ class zmq_pub net::zmq::socket relay_; std::deque<std::vector<txpool_event>> txes_; std::array<std::size_t, 2> chain_subs_; + std::array<std::size_t, 1> miner_subs_; std::array<std::size_t, 2> txpool_subs_; boost::mutex sync_; //!< Synchronizes counts in `*_subs_` arrays. @@ -88,6 +90,11 @@ class zmq_pub \return Number of ZMQ messages sent to relay. */ std::size_t send_chain_main(std::uint64_t height, epee::span<const cryptonote::block> blocks); + /*! Send a `ZMQ_PUB` notification for a new miner data. + Thread-safe. + \return Number of ZMQ messages sent to relay. */ + std::size_t send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog); + /*! Send a `ZMQ_PUB` notification for new tx(es) being added to the local pool. Thread-safe. \return Number of ZMQ messages sent to relay. */ @@ -100,6 +107,13 @@ class zmq_pub void operator()(std::uint64_t height, epee::span<const cryptonote::block> blocks) const; }; + //! Callable for `send_miner_data` with weak ownership to `zmq_pub` object. + struct miner_data + { + std::weak_ptr<zmq_pub> self_; + void operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const; + }; + //! Callable for `send_txpool_add` with weak ownership to `zmq_pub` object. struct txpool_add { diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 28e207ff2..b03da1edc 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -34,6 +34,7 @@ #include <type_traits> #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_core/cryptonote_tx_utils.h" // drop macro from windows.h #ifdef GetObject @@ -1411,6 +1412,27 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribu GET_FROM_JSON_OBJECT(val, dist.data.base, base); } +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry) +{ + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, id, entry.id); + INSERT_INTO_JSON_OBJECT(dest, weight, entry.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, entry.fee); + dest.EndObject(); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, entry.id, id); + GET_FROM_JSON_OBJECT(val, entry.weight, weight); + GET_FROM_JSON_OBJECT(val, entry.fee, fee); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 35ea990b3..c858faf5a 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry); +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index dc031b36c..fd784c5ae 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -6061,6 +6061,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args auto local_args = args; LOCK_IDLE_SCOPE(); + std::set<uint32_t> subaddr_indices; bool filter = false; bool available = false; bool verbose = false; @@ -6086,6 +6087,11 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args verbose = true; else if (local_args[0] == "uses") uses = true; + else if (local_args[0].substr(0, 6) == "index=") + { + if (!parse_subaddress_indices(local_args[0], subaddr_indices)) + return true; + } else { fail_msg_writer() << tr("Invalid keyword: ") << local_args.front(); @@ -6098,14 +6104,6 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args PAUSE_READLINE(); - std::set<uint32_t> subaddr_indices; - if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") - { - if (!parse_subaddress_indices(local_args[0], subaddr_indices)) - return true; - local_args.erase(local_args.begin()); - } - if (local_args.size() > 0) { PRINT_USAGE(USAGE_INCOMING_TRANSFERS); diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 0afbda705..989061250 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -450,7 +450,7 @@ WalletImpl::~WalletImpl() LOG_PRINT_L1(__FUNCTION__); m_wallet->callback(NULL); // Pause refresh thread - prevents refresh from starting again - pauseRefresh(); + WalletImpl::pauseRefresh(); // Call the method directly (not polymorphically) to protect against UB in destructor. // Close wallet - stores cache and stops ongoing refresh operation close(false); // do not store wallet as part of the closing activities // Stop refresh thread diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 417a27db5..f5d5e2168 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -49,6 +49,11 @@ namespace epee { namespace Monero { +WalletManagerImpl::WalletManagerImpl() +{ + tools::set_strict_default_file_permissions(true); +} + Wallet *WalletManagerImpl::createWallet(const std::string &path, const std::string &password, const std::string &language, NetworkType nettype, uint64_t kdf_rounds) { diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index cf3056a17..1e8cff877 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -95,7 +95,7 @@ public: bool setProxy(const std::string &address) override; private: - WalletManagerImpl() {} + WalletManagerImpl(); friend struct WalletManagerFactory; net::http::client m_http_client; std::string m_errorString; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 5a4cafc32..2a190add5 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -146,7 +146,7 @@ using namespace cryptonote; #define IGNORE_LONG_PAYMENT_ID_FROM_BLOCK_VERSION 12 #define DEFAULT_UNLOCK_TIME (CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE * DIFFICULTY_TARGET_V2) -#define RECENT_SPEND_WINDOW (50 * DIFFICULTY_TARGET_V2) +#define RECENT_SPEND_WINDOW (15 * DIFFICULTY_TARGET_V2) static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string MULTISIG_EXTRA_INFO_MAGIC = "MultisigxV1"; @@ -314,7 +314,6 @@ void do_prepare_file_names(const std::string& file_path, std::string& keys_file, { keys_file = file_path; wallet_file = file_path; - boost::system::error_code e; if(string_tools::get_extension(keys_file) == "keys") {//provided keys file name wallet_file = string_tools::cut_off_extension(wallet_file); @@ -1024,13 +1023,7 @@ gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets, double shap end = rct_offsets.data() + rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE; num_rct_outputs = *(end - 1); THROW_WALLET_EXCEPTION_IF(num_rct_outputs == 0, error::wallet_internal_error, "No rct outputs"); - THROW_WALLET_EXCEPTION_IF(outputs_to_consider == 0, error::wallet_internal_error, "No rct outputs to consider"); - average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / outputs_to_consider; // this assumes constant target over the whole rct range - if (average_output_time == 0) { - // TODO: apply this to all cases; do so alongside a hard fork, where all clients will update at the same time, preventing anonymity puddle formation - average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / static_cast<double>(outputs_to_consider); - } - THROW_WALLET_EXCEPTION_IF(average_output_time == 0, error::wallet_internal_error, "Average seconds per output cannot be 0."); + average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / static_cast<double>(outputs_to_consider); // this assumes constant target over the whole rct range }; gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets): gamma_picker(rct_offsets, GAMMA_SHAPE, GAMMA_SCALE) {} @@ -1235,8 +1228,6 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std m_ring_history_saved(false), m_ringdb(), m_last_block_reward(0), - m_encrypt_keys_after_refresh(boost::none), - m_decrypt_keys_lockers(0), m_unattended(unattended), m_devices_registered(false), m_device_last_key_image_sync(0), @@ -1888,8 +1879,7 @@ void wallet2::scan_output(const cryptonote::transaction &tx, bool miner_tx, cons boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password(pool ? "output found in pool" : "output received"); THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); - decrypt_keys(*pwd); - m_encrypt_keys_after_refresh = *pwd; + m_encrypt_keys_after_refresh.reset(new wallet_keys_unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, *pwd)); } } @@ -3021,11 +3011,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, MTRACE("update_pool_state start"); auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { - if (m_encrypt_keys_after_refresh) - { - encrypt_keys(*m_encrypt_keys_after_refresh); - m_encrypt_keys_after_refresh = boost::none; - } + m_encrypt_keys_after_refresh.reset(); }); // get the pool state @@ -3456,11 +3442,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo start_height = 0; auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { - if (m_encrypt_keys_after_refresh) - { - encrypt_keys(*m_encrypt_keys_after_refresh); - m_encrypt_keys_after_refresh = boost::none; - } + m_encrypt_keys_after_refresh.reset(); }); auto scope_exit_handler_hwdev = epee::misc_utils::create_scope_leave_handler([&](){hwdev.computing_key_images(false);}); @@ -4467,7 +4449,26 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_account.set_device(hwdev); account_public_address device_account_public_address; - THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + bool fetch_device_address = true; + + ::hw::device_cold* dev_cold = nullptr; + if (m_key_device_type == hw::device::device_type::TREZOR && (dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev)) != nullptr) { + THROW_WALLET_EXCEPTION_IF(!dev_cold->get_public_address_with_no_passphrase(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + if (device_account_public_address == m_account.get_keys().m_account_address) { + LOG_PRINT_L0("Wallet opened with an empty passphrase"); + fetch_device_address = false; + dev_cold->set_use_empty_passphrase(true); + } else { + fetch_device_address = true; + LOG_PRINT_L0("Wallet opening with an empty passphrase failed. Retry again: " << fetch_device_address); + dev_cold->reset_session(); + } + } + + if (fetch_device_address) { + THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + } + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. If the device uses the passphrase feature, please check whether the passphrase was entered correctly (it may have been misspelled - different passphrases generate different wallets, passphrase is case-sensitive). " "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + ", wallet address: " + m_account.get_public_address_str(m_nettype)); @@ -4581,18 +4582,12 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip void wallet2::encrypt_keys(const crypto::chacha_key &key) { - boost::lock_guard<boost::mutex> lock(m_decrypt_keys_lock); - if (--m_decrypt_keys_lockers) // another lock left ? - return; m_account.encrypt_keys(key); m_account.decrypt_viewkey(key); } void wallet2::decrypt_keys(const crypto::chacha_key &key) { - boost::lock_guard<boost::mutex> lock(m_decrypt_keys_lock); - if (m_decrypt_keys_lockers++) // already unlocked ? - return; m_account.encrypt_viewkey(key); m_account.decrypt_keys(key); } @@ -7077,7 +7072,6 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func) { std::string s = signed_tx_st; - boost::system::error_code errcode; signed_tx_set signed_txs; const size_t magiclen = strlen(SIGNED_TX_PREFIX) - 1; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index facf9878d..7648becc8 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1798,9 +1798,7 @@ private: crypto::secret_key m_original_view_secret_key; crypto::chacha_key m_cache_key; - boost::optional<epee::wipeable_string> m_encrypt_keys_after_refresh; - boost::mutex m_decrypt_keys_lock; - unsigned int m_decrypt_keys_lockers; + std::shared_ptr<wallet_keys_unlocker> m_encrypt_keys_after_refresh; bool m_unattended; bool m_devices_registered; diff --git a/src/wallet/wallet_rpc_helpers.h b/src/wallet/wallet_rpc_helpers.h index 35714db03..6f50b6727 100644 --- a/src/wallet/wallet_rpc_helpers.h +++ b/src/wallet/wallet_rpc_helpers.h @@ -28,6 +28,7 @@ #pragma once +#include <limits> #include <type_traits> namespace diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index 30d6f8133..069bc19b2 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -265,11 +265,17 @@ namespace virtual void callback(cryptonote::levin::detail::p2p_context& context) override final {} - virtual void on_connection_new(cryptonote::levin::detail::p2p_context&) override final - {} + virtual void on_connection_new(cryptonote::levin::detail::p2p_context& context) override final + { + if (notifier) + notifier->on_handshake_complete(context.m_connection_id, context.m_is_income); + } - virtual void on_connection_close(cryptonote::levin::detail::p2p_context&) override final - {} + virtual void on_connection_close(cryptonote::levin::detail::p2p_context& context) override final + { + if (notifier) + notifier->on_connection_close(context.m_connection_id); + } public: test_receiver() @@ -306,6 +312,8 @@ namespace { return get_raw_message(notified_); } + + std::shared_ptr<cryptonote::levin::notify> notifier{}; }; class levin_notify : public ::testing::Test @@ -343,13 +351,16 @@ namespace EXPECT_EQ(connection_ids_.size(), connections_->get_connections_count()); } - cryptonote::levin::notify make_notifier(const std::size_t noise_size, bool is_public, bool pad_txs) + std::shared_ptr<cryptonote::levin::notify> make_notifier(const std::size_t noise_size, bool is_public, bool pad_txs) { epee::byte_slice noise = nullptr; if (noise_size) noise = epee::levin::make_noise_notify(noise_size); epee::net_utils::zone zone = is_public ? epee::net_utils::zone::public_ : epee::net_utils::zone::i2p; - return cryptonote::levin::notify{io_service_, connections_, std::move(noise), zone, pad_txs, events_}; + receiver_.notifier.reset( + new cryptonote::levin::notify{io_service_, connections_, std::move(noise), zone, pad_txs, events_} + ); + return receiver_.notifier; } boost::uuids::random_generator random_generator_; @@ -590,7 +601,8 @@ TEST_F(levin_notify, defaulted) TEST_F(levin_notify, fluff_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -636,7 +648,8 @@ TEST_F(levin_notify, fluff_without_padding) TEST_F(levin_notify, stem_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -708,7 +721,8 @@ TEST_F(levin_notify, stem_without_padding) TEST_F(levin_notify, stem_no_outs_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(true); @@ -764,7 +778,8 @@ TEST_F(levin_notify, stem_no_outs_without_padding) TEST_F(levin_notify, local_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -836,7 +851,8 @@ TEST_F(levin_notify, local_without_padding) TEST_F(levin_notify, forward_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -908,7 +924,8 @@ TEST_F(levin_notify, forward_without_padding) TEST_F(levin_notify, block_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -937,7 +954,8 @@ TEST_F(levin_notify, block_without_padding) TEST_F(levin_notify, none_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -966,7 +984,8 @@ TEST_F(levin_notify, none_without_padding) TEST_F(levin_notify, fluff_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1012,7 +1031,8 @@ TEST_F(levin_notify, fluff_with_padding) TEST_F(levin_notify, stem_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1079,7 +1099,8 @@ TEST_F(levin_notify, stem_with_padding) TEST_F(levin_notify, stem_no_outs_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(true); @@ -1135,7 +1156,8 @@ TEST_F(levin_notify, stem_no_outs_with_padding) TEST_F(levin_notify, local_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1202,7 +1224,8 @@ TEST_F(levin_notify, local_with_padding) TEST_F(levin_notify, forward_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1269,7 +1292,8 @@ TEST_F(levin_notify, forward_with_padding) TEST_F(levin_notify, block_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1298,7 +1322,8 @@ TEST_F(levin_notify, block_with_padding) TEST_F(levin_notify, none_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, true, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1327,7 +1352,8 @@ TEST_F(levin_notify, none_with_padding) TEST_F(levin_notify, private_fluff_without_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1378,7 +1404,8 @@ TEST_F(levin_notify, private_fluff_without_padding) TEST_F(levin_notify, private_stem_without_padding) { // private mode always uses fluff but marked as stem - cryptonote::levin::notify notifier = make_notifier(0, false, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1429,7 +1456,8 @@ TEST_F(levin_notify, private_stem_without_padding) TEST_F(levin_notify, private_local_without_padding) { // private mode always uses fluff but marked as stem - cryptonote::levin::notify notifier = make_notifier(0, false, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1480,7 +1508,8 @@ TEST_F(levin_notify, private_local_without_padding) TEST_F(levin_notify, private_forward_without_padding) { // private mode always uses fluff but marked as stem - cryptonote::levin::notify notifier = make_notifier(0, false, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1531,7 +1560,8 @@ TEST_F(levin_notify, private_forward_without_padding) TEST_F(levin_notify, private_block_without_padding) { // private mode always uses fluff but marked as stem - cryptonote::levin::notify notifier = make_notifier(0, false, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1561,7 +1591,8 @@ TEST_F(levin_notify, private_block_without_padding) TEST_F(levin_notify, private_none_without_padding) { // private mode always uses fluff but marked as stem - cryptonote::levin::notify notifier = make_notifier(0, false, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1590,7 +1621,8 @@ TEST_F(levin_notify, private_none_without_padding) TEST_F(levin_notify, private_fluff_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1640,7 +1672,8 @@ TEST_F(levin_notify, private_fluff_with_padding) TEST_F(levin_notify, private_stem_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1690,7 +1723,8 @@ TEST_F(levin_notify, private_stem_with_padding) TEST_F(levin_notify, private_local_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1740,7 +1774,8 @@ TEST_F(levin_notify, private_local_with_padding) TEST_F(levin_notify, private_forward_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1790,7 +1825,8 @@ TEST_F(levin_notify, private_forward_with_padding) TEST_F(levin_notify, private_block_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1819,7 +1855,8 @@ TEST_F(levin_notify, private_block_with_padding) TEST_F(levin_notify, private_none_with_padding) { - cryptonote::levin::notify notifier = make_notifier(0, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, false, true); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < 10; ++count) add_connection(count % 2 == 0); @@ -1850,7 +1887,8 @@ TEST_F(levin_notify, stem_mappings) { static constexpr const unsigned test_connections_count = (CRYPTONOTE_DANDELIONPP_STEMS + 1) * 2; - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < test_connections_count; ++count) add_connection(count % 2 == 0); @@ -1973,7 +2011,8 @@ TEST_F(levin_notify, fluff_multiple) { static constexpr const unsigned test_connections_count = (CRYPTONOTE_DANDELIONPP_STEMS + 1) * 2; - cryptonote::levin::notify notifier = make_notifier(0, true, false); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(0, true, false); + auto ¬ifier = *notifier_ptr; for (unsigned count = 0; count < test_connections_count; ++count) add_connection(count % 2 == 0); @@ -2091,7 +2130,8 @@ TEST_F(levin_notify, noise) txs[0].resize(1900, 'h'); const boost::uuids::uuid incoming_id = random_generator_(); - cryptonote::levin::notify notifier = make_notifier(2048, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(2048, false, true); + auto ¬ifier = *notifier_ptr; { const auto status = notifier.get_status(); @@ -2182,7 +2222,8 @@ TEST_F(levin_notify, noise_stem) txs[0].resize(1900, 'h'); const boost::uuids::uuid incoming_id = random_generator_(); - cryptonote::levin::notify notifier = make_notifier(2048, false, true); + std::shared_ptr<cryptonote::levin::notify> notifier_ptr = make_notifier(2048, false, true); + auto ¬ifier = *notifier_ptr; { const auto status = notifier.get_status(); diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp index f075034bd..c0b057c9a 100644 --- a/tests/unit_tests/long_term_block_weight.cpp +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -106,10 +106,16 @@ static uint32_t lcg() } +struct BlockchainAndPool +{ + cryptonote::tx_memory_pool txpool; + cryptonote::Blockchain bc; + BlockchainAndPool(): txpool(bc), bc(txpool) {} +}; + #define PREFIX_WINDOW(hf_version,window) \ - std::unique_ptr<cryptonote::Blockchain> bc; \ - cryptonote::tx_memory_pool txpool(*bc); \ - bc.reset(new cryptonote::Blockchain(txpool)); \ + BlockchainAndPool bap; \ + cryptonote::Blockchain *bc = &bap.bc; \ struct get_test_options { \ const std::pair<uint8_t, uint64_t> hard_forks[3]; \ const cryptonote::test_options test_options = { \ @@ -118,8 +124,7 @@ static uint32_t lcg() }; \ get_test_options(): hard_forks{std::make_pair(1, (uint64_t)0), std::make_pair((uint8_t)hf_version, (uint64_t)1), std::make_pair((uint8_t)0, (uint64_t)0)} {} \ } opts; \ - cryptonote::Blockchain *blockchain = bc.get(); \ - bool r = blockchain->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0, NULL); \ + bool r = bc->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0, NULL); \ ASSERT_TRUE(r) #define PREFIX(hf_version) PREFIX_WINDOW(hf_version, TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW) diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index cab600b3d..7907e9a9a 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -35,6 +35,7 @@ #include "cryptonote_core/i_core_events.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "cryptonote_protocol/cryptonote_protocol_handler.inl" +#include <condition_variable> #define MAKE_IPV4_ADDRESS(a,b,c,d) epee::net_utils::ipv4_network_address{MAKE_IP(a,b,c,d),0} #define MAKE_IPV4_ADDRESS_PORT(a,b,c,d,e) epee::net_utils::ipv4_network_address{MAKE_IP(a,b,c,d),e} @@ -303,6 +304,9 @@ TEST(node_server, bind_same_p2p_port) Relevant part about REUSEADDR from man: https://www.man7.org/linux/man-pages/man7/ip.7.html + + For Mac OSX, set the following alias, before running the test, or else it will fail: + sudo ifconfig lo0 alias 127.0.0.2 */ vm.find(nodetool::arg_p2p_bind_ip.name)->second = boost::program_options::variable_value(std::string("127.0.0.2"), false); vm.find(nodetool::arg_p2p_bind_port.name)->second = boost::program_options::variable_value(std::string(port), false); @@ -906,5 +910,264 @@ TEST(cryptonote_protocol_handler, race_condition) remove_tree(dir); } +TEST(node_server, race_condition) +{ + struct contexts { + using cryptonote = cryptonote::cryptonote_connection_context; + using p2p = nodetool::p2p_connection_context_t<cryptonote>; + }; + using context_t = contexts::cryptonote; + using options_t = boost::program_options::variables_map; + using options_description_t = boost::program_options::options_description; + using worker_t = std::thread; + struct protocol_t { + private: + using p2p_endpoint_t = nodetool::i_p2p_endpoint<context_t>; + using lock_t = std::mutex; + using condition_t = std::condition_variable_any; + using unique_lock_t = std::unique_lock<lock_t>; + p2p_endpoint_t *p2p_endpoint; + lock_t lock; + condition_t condition; + bool started{}; + size_t counter{}; + public: + using payload_t = cryptonote::CORE_SYNC_DATA; + using blob_t = cryptonote::blobdata; + using connection_context = context_t; + using payload_type = payload_t; + using relay_t = cryptonote::relay_method; + using string_t = std::string; + using span_t = epee::span<const uint8_t>; + using blobs_t = epee::span<const cryptonote::blobdata>; + using connections_t = std::list<cryptonote::connection_info>; + using block_queue_t = cryptonote::block_queue; + using stripes_t = std::pair<uint32_t, uint32_t>; + using byte_stream_t = epee::byte_stream; + struct core_events_t: cryptonote::i_core_events { + uint64_t get_current_blockchain_height() const override { return {}; } + bool is_synchronized() const override { return {}; } + void on_transactions_relayed(blobs_t blobs, relay_t relay) override {} + }; + int handle_invoke_map(bool is_notify, int command, const span_t in, byte_stream_t &out, context_t &context, bool &handled) { + return {}; + } + bool on_idle() { + if (not p2p_endpoint) + return {}; + { + unique_lock_t guard(lock); + if (not started) + started = true; + else + return {}; + } + std::vector<blob_t> txs(128 / 64 * 1024 * 1024, blob_t(1, 'x')); + worker_t worker([this]{ + p2p_endpoint->for_each_connection( + [this](context_t &, uint64_t, uint32_t){ + { + unique_lock_t guard(lock); + ++counter; + condition.notify_all(); + condition.wait(guard, [this]{ return counter >= 3; }); + } + std::this_thread::sleep_for(std::chrono::milliseconds(8)); + return false; + } + ); + }); + { + unique_lock_t guard(lock); + ++counter; + condition.notify_all(); + condition.wait(guard, [this]{ return counter >= 3; }); + ++counter; + condition.notify_all(); + condition.wait(guard, [this]{ return counter >= 5; }); + } + p2p_endpoint->send_txs( + std::move(txs), + epee::net_utils::zone::public_, + {}, + relay_t::fluff + ); + worker.join(); + return {}; + } + bool init(const options_t &options) { return {}; } + bool deinit() { return {}; } + void set_p2p_endpoint(p2p_endpoint_t *p2p_endpoint) { + this->p2p_endpoint = p2p_endpoint; + } + bool process_payload_sync_data(const payload_t &payload, contexts::p2p &context, bool is_inital) { + context.m_state = context_t::state_normal; + context.m_needed_objects.resize(512 * 1024); + { + unique_lock_t guard(lock); + ++counter; + condition.notify_all(); + condition.wait(guard, [this]{ return counter >= 3; }); + ++counter; + condition.notify_all(); + condition.wait(guard, [this]{ return counter >= 5; }); + } + return true; + } + bool get_payload_sync_data(blob_t &blob) { return {}; } + bool get_payload_sync_data(payload_t &payload) { return {}; } + bool on_callback(context_t &context) { return {}; } + core_events_t &get_core(){ static core_events_t core_events; return core_events;} + void log_connections() {} + connections_t get_connections() { return {}; } + const block_queue_t &get_block_queue() const { + static block_queue_t block_queue; + return block_queue; + } + void stop() {} + void on_connection_close(context_t &context) {} + void set_max_out_peers(unsigned int max) {} + bool no_sync() const { return {}; } + void set_no_sync(bool value) {} + string_t get_peers_overview() const { return {}; } + stripes_t get_next_needed_pruning_stripe() const { return {}; } + bool needs_new_sync_connections() const { return {}; } + bool is_busy_syncing() { return {}; } + }; + using node_server_t = nodetool::node_server<protocol_t>; + auto conduct_test = [](protocol_t &protocol){ + struct messages { + struct core { + using sync = cryptonote::CORE_SYNC_DATA; + }; + using handshake = nodetool::COMMAND_HANDSHAKE_T<core::sync>; + }; + using handler_t = epee::levin::async_protocol_handler<context_t>; + using connection_t = epee::net_utils::connection<handler_t>; + using connection_ptr = boost::shared_ptr<connection_t>; + using shared_state_t = typename connection_t::shared_state; + using shared_state_ptr = std::shared_ptr<shared_state_t>; + using io_context_t = boost::asio::io_service; + using work_t = boost::asio::io_service::work; + using work_ptr = std::shared_ptr<work_t>; + using workers_t = std::vector<std::thread>; + using endpoint_t = boost::asio::ip::tcp::endpoint; + using event_t = epee::simple_event; + struct command_handler_t: epee::levin::levin_commands_handler<context_t> { + using span_t = epee::span<const uint8_t>; + using byte_stream_t = epee::byte_stream; + int invoke(int, const span_t, byte_stream_t &, context_t &) override { return {}; } + int notify(int, const span_t, context_t &) override { return {}; } + void callback(context_t &) override {} + void on_connection_new(context_t &) override {} + void on_connection_close(context_t &) override {} + ~command_handler_t() override {} + static void destroy(epee::levin::levin_commands_handler<context_t>* ptr) { delete ptr; } + }; + io_context_t io_context; + work_ptr work = std::make_shared<work_t>(io_context); + workers_t workers; + while (workers.size() < 4) { + workers.emplace_back([&io_context]{ + io_context.run(); + }); + } + io_context.post([&]{ + protocol.on_idle(); + }); + io_context.post([&]{ + protocol.on_idle(); + }); + shared_state_ptr shared_state = std::make_shared<shared_state_t>(); + shared_state->set_handler(new command_handler_t, &command_handler_t::destroy); + connection_ptr conn{new connection_t(io_context, shared_state, {}, {})}; + endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 48080); + conn->socket().connect(endpoint); + conn->socket().set_option(boost::asio::ip::tcp::socket::reuse_address(true)); + conn->start({}, {}); + context_t context; + conn->get_context(context); + event_t handshaked; + typename messages::handshake::request_t msg{{ + ::config::NETWORK_ID, + 58080, + }}; + epee::net_utils::async_invoke_remote_command2<typename messages::handshake::response>( + context, + messages::handshake::ID, + msg, + *shared_state, + [conn, &handshaked](int code, const typename messages::handshake::response &msg, context_t &context){ + EXPECT_TRUE(code >= 0); + handshaked.raise(); + }, + P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT + ); + handshaked.wait(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + conn.reset(); + work.reset(); + for (auto& w: workers) { + w.join(); + } + }; + using path_t = boost::filesystem::path; + using ec_t = boost::system::error_code; + auto create_dir = []{ + ec_t ec; + path_t path = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path("daemon-%%%%%%%%%%%%%%%%", ec); + if (ec) + return path_t{}; + auto success = boost::filesystem::create_directory(path, ec); + if (not ec && success) + return path; + return path_t{}; + }; + auto remove_tree = [](const path_t &path){ + ec_t ec; + boost::filesystem::remove_all(path, ec); + }; + const auto dir = create_dir(); + ASSERT_TRUE(not dir.empty()); + protocol_t protocol{}; + node_server_t node_server(protocol); + protocol.set_p2p_endpoint(&node_server); + node_server.init( + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--p2p-bind-ip=127.0.0.1", + "--p2p-bind-port=48080", + "--out-peers=0", + "--data-dir", + dir.string(), + "--no-igd", + "--add-exclusive-node=127.0.0.1:48080", + "--check-updates=disabled", + "--disable-dns-checkpoints", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + node_server_t::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }() + ); + worker_t worker([&]{ + node_server.run(); + }); + conduct_test(protocol); + node_server.send_stop_signal(); + worker.join(); + node_server.deinit(); + remove_tree(dir); +} + namespace nodetool { template class node_server<cryptonote::t_cryptonote_protocol_handler<test_core>>; } namespace cryptonote { template class t_cryptonote_protocol_handler<test_core>; } diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 535752665..f4c73d3d5 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -132,7 +132,8 @@ TEST(Serialization, BinaryArchiveInts) { ASSERT_EQ(8, oss.str().size()); ASSERT_EQ(string("\0\0\0\0\xff\0\0\0", 8), oss.str()); - binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; + const std::string s = oss.str(); + binary_archive<false> iar{epee::strspan<std::uint8_t>(s)}; iar.serialize_int(x1); ASSERT_EQ(8, iar.getpos()); ASSERT_TRUE(iar.good()); @@ -150,7 +151,8 @@ TEST(Serialization, BinaryArchiveVarInts) { ASSERT_EQ(6, oss.str().size()); ASSERT_EQ(string("\x80\x80\x80\x80\xF0\x1F", 6), oss.str()); - binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; + const std::string s = oss.str(); + binary_archive<false> iar{epee::strspan<std::uint8_t>(s)}; iar.serialize_varint(x1); ASSERT_TRUE(iar.good()); ASSERT_EQ(x, x1); diff --git a/utils/health/README.md b/utils/health/README.md index 05eb057a1..d4ca4a538 100644 --- a/utils/health/README.md +++ b/utils/health/README.md @@ -16,7 +16,8 @@ On the first run, the script will complain about the missing ClangBuildAnalyzer ## clang-tidy -`utils/health/clang-tidy-run.sh` +`utils/health/clang-tidy-run-cc.sh` +`utils/health/clang-tidy-run-cpp.sh` Performs Lint checks on the source code and stores the result in the build directory. More information on the [home page](https://clang.llvm.org/extra/clang-tidy/). ## include-what-you-use diff --git a/utils/health/clang-tidy-run-cc.sh b/utils/health/clang-tidy-run-cc.sh new file mode 100755 index 000000000..0cf737131 --- /dev/null +++ b/utils/health/clang-tidy-run-cc.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2021, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# clang-tidy runs lint checks on C & C++ sources and headers. +# Run this script from the source directory. + +DIR_THIS="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$DIR_THIS/clang-tidy-run-common.sh" + +tidy_for_language "C" diff --git a/utils/health/clang-tidy-run.sh b/utils/health/clang-tidy-run-common.sh index 6b34f6a3b..a086821a6 100755 --- a/utils/health/clang-tidy-run.sh +++ b/utils/health/clang-tidy-run-common.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -# Copyright (c) 2014-2020, The Monero Project +# Copyright (c) 2014-2021, The Monero Project # # All rights reserved. # @@ -29,7 +29,7 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # clang-tidy runs lint checks on C & C++ sources and headers. -# Run this script from the source directory. +# Don't use this script directly but call clang-tidy-run-cc.sh or clang-tidy-run-cpp.sh instead DIR_BUILD_BASE="build/clang-tidy" RESULT_BASE="clang-tidy-result" @@ -40,6 +40,7 @@ function tidy_for_language() { RESULT="${RESULT_BASE}-${LANG}.txt" mkdir -p "$DIR_BUILD" && pushd "$DIR_BUILD" + rm `find . -name "CMakeCache.txt"` || true cmake ../.. \ -DCMAKE_C_COMPILER=clang \ @@ -52,14 +53,13 @@ function tidy_for_language() { make clean # Clean up, so that the result can be regenerated from scratch time make -k 2>&1 | tee "$RESULT" # Build and store the result. -k means: ignore errors #time make -k easylogging 2>&1 | tee "$RESULT" # Quick testing: build a single target - gzip -f "$RESULT" # Zip the result, because it's huge. -f overwrites the previously generated result - + KPI=$(cat "$RESULT" | wc -l) + tar -cJvf "$RESULT.txz" "$RESULT" # Zip the result, because it's huge. + rm -v "$RESULT" echo "" - echo "Readable result stored in: $DIR_BUILD/$RESULT.gz" + echo "Readable result stored in: $DIR_BUILD/$RESULT.txz" + + echo "$KPI" > "kpis.txt" popd } - -tidy_for_language "C" -tidy_for_language "CXX" - diff --git a/utils/health/clang-tidy-run-cpp.sh b/utils/health/clang-tidy-run-cpp.sh new file mode 100755 index 000000000..7cf08fe78 --- /dev/null +++ b/utils/health/clang-tidy-run-cpp.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2021, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# clang-tidy runs lint checks on C & C++ sources and headers. +# Run this script from the source directory. + +DIR_THIS="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$DIR_THIS/clang-tidy-run-common.sh" + +tidy_for_language "CXX" diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 207565e6f..397000b68 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -53,6 +53,28 @@ class Daemon(object): return self.rpc.send_json_rpc_request(getblocktemplate) get_block_template = getblocktemplate + def get_miner_data(self): + get_miner_data = { + 'method': 'get_miner_data', + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_miner_data) + + def calc_pow(self, major_version, height, block_blob, seed_hash = ''): + calc_pow = { + 'method': 'calc_pow', + 'params': { + 'major_version': major_version, + 'height': height, + 'block_blob' : block_blob, + 'seed_hash' : seed_hash, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(calc_pow) + def add_aux_pow(self, blocktemplate_blob, aux_pow, client = ""): add_aux_pow = { 'method': 'add_aux_pow', |