diff options
72 files changed, 785 insertions, 369 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c28a71ce..b97be58b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,19 +6,20 @@ on: [push, pull_request] env: REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost BUILD_DEFAULT_LINUX: | - ccache --max-size=150M cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release && cmake --build build -j3 APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache' APT_SET_CONF: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + CCACHE_SETTINGS: | + ccache --max-size=150M + ccache --set-config=compression=true jobs: build-macos: runs-on: macOS-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 @@ -27,19 +28,18 @@ jobs: - uses: actions/cache@v2 with: path: /Users/runner/Library/Caches/ccache - key: ccache-macos-build-${{ github.sha }} - restore-keys: ccache-macos-build- + key: ccache-${{ runner.os }}-build-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build- - name: install dependencies - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} make -j3 build-windows: runs-on: windows-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp CCACHE_DIR: C:\Users\runneradmin\.ccache defaults: @@ -52,15 +52,15 @@ jobs: - uses: actions/cache@v2 with: path: C:\Users\runneradmin\.ccache - key: ccache-windows-build-${{ github.sha }} - restore-keys: ccache-windows-build- + key: ccache-${{ runner.os }}-build-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build- - uses: eine/setup-msys2@v2 with: update: true - install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} make release-static-win64 -j2 # See the OS labels and monitor deprecations here: @@ -69,7 +69,6 @@ jobs: build-ubuntu: runs-on: ${{ matrix.os }} env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp strategy: matrix: @@ -81,8 +80,8 @@ jobs: - uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-build-${{ matrix.os }}-${{ github.sha }} - restore-keys: ccache-ubuntu-build-${{ matrix.os }} + key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }} - name: remove bundled boost run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf @@ -92,12 +91,13 @@ jobs: - name: install monero dependencies run: ${{env.APT_INSTALL_LINUX}} - name: build - run: ${{env.BUILD_DEFAULT_LINUX}} + run: | + ${{env.CCACHE_SETTINGS}} + ${{env.BUILD_DEFAULT_LINUX}} libwallet-ubuntu: runs-on: ubuntu-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 @@ -106,8 +106,8 @@ jobs: - uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-libwallet-${{ github.sha }} - restore-keys: ccache-ubuntu-libwallet- + key: ccache-${{ runner.os }}-libwallet-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-libwallet- - name: remove bundled boost run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf @@ -118,7 +118,7 @@ jobs: run: ${{env.APT_INSTALL_LINUX}} - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} cmake . make wallet_api -j3 @@ -126,7 +126,6 @@ jobs: needs: build-ubuntu runs-on: ubuntu-latest env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 @@ -136,8 +135,8 @@ jobs: uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-build-ubuntu-latest-${{ github.sha }} - restore-keys: ccache-ubuntu-build-ubuntu-latest + key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest - name: remove bundled boost run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf @@ -152,6 +151,7 @@ jobs: env: CTEST_OUTPUT_ON_FAILURE: ON run: | + ${{env.CCACHE_SETTINGS}} ${{env.BUILD_DEFAULT_LINUX}} cmake --build build --target test diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index 8e4eaf177..74d0ceabc 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -2,11 +2,19 @@ name: ci/gh-actions/depends on: [push, pull_request] +env: + APT_SET_CONF: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + CCACHE_SETTINGS: | + ccache --max-size=150M + ccache --set-config=compression=true + jobs: build-macos: runs-on: ubuntu-18.04 env: - CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp strategy: fail-fast: false @@ -69,10 +77,7 @@ jobs: key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} restore-keys: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: install dependencies run: sudo apt update; sudo apt -y install build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache ${{ matrix.toolchain.packages }} - name: prepare apple-darwin11 @@ -88,7 +93,7 @@ jobs: sudo update-alternatives --set ${{ matrix.toolchain.host }}-gcc $(which ${{ matrix.toolchain.host }}-gcc-posix) - name: build run: | - ccache --max-size=150M + ${{env.CCACHE_SETTINGS}} make depends target=${{ matrix.toolchain.host }} -j2 - uses: actions/upload-artifact@v2 if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} diff --git a/.gitmodules b/.gitmodules index 3cf831bcd..721cce3b4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "external/unbound"] - path = external/unbound - url = https://github.com/monero-project/unbound - branch = monero [submodule "external/miniupnp"] path = external/miniupnp url = https://github.com/miniupnp/miniupnp diff --git a/CMakeLists.txt b/CMakeLists.txt index f8af45f4b..bb019e178 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,14 @@ endif() enable_language(C ASM) +# Require C11/C++11 and disable extensions for all targets +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + function (die msg) if (NOT WIN32) string(ASCII 27 Esc) @@ -106,6 +114,8 @@ function (die msg) endfunction () function (add_c_flag_if_supported flag var) + # Prepending the flag with -Werror will only add the flag, + # if it doesn't result in generation of a warning of using a flag unknown to the compiler. set(TMP "-Werror ${flag}") string(REGEX REPLACE "[- ]" "_" supported ${TMP}_c) check_c_compiler_flag(${TMP} ${supported}) @@ -345,7 +355,6 @@ if(NOT MANUAL_SUBMODULES) message(STATUS "Checking submodules") check_submodule(external/miniupnp) - check_submodule(external/unbound) check_submodule(external/rapidjson) check_submodule(external/trezor-common) check_submodule(external/randomx) @@ -383,18 +392,6 @@ endif() message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) - -if (NOT DEFINED ENV{DEVELOPER_LIBUNBOUND_OLD}) - message(STATUS "Could not find DEVELOPER_LIBUNBOUND_OLD in env (not required)") -elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 1) - message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 1, will use the work around") - add_definitions(-DDEVELOPER_LIBUNBOUND_OLD) -elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 0) - message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 0") -else() - message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD with bad value. Will NOT use the work around") -endif() - set_property(GLOBAL PROPERTY USE_FOLDERS ON) enable_testing() @@ -502,6 +499,7 @@ if(STATIC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DZMQ_STATIC") endif() +option(SANITIZE "Use ASAN memory sanitizer" OFF) if(SANITIZE) if (MSVC) message(FATAL_ERROR "Cannot sanitize with MSVC") @@ -570,7 +568,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "(SunOS|Solaris)") endif () if (APPLE AND NOT IOS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default") if (NOT OPENSSL_ROOT_DIR) EXECUTE_PROCESS(COMMAND brew --prefix openssl OUTPUT_VARIABLE OPENSSL_ROOT_DIR @@ -618,8 +616,6 @@ if (NOT (MSVC OR ARCH)) set_default_arch() endif() -CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) - option(COVERAGE "Enable profiling for test coverage report" OFF) if(COVERAGE) message(STATUS "Building with profiling for test coverage report") @@ -672,8 +668,7 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}/translations") add_subdirectory(external) # Final setup for libunbound -include_directories(${UNBOUND_INCLUDE}) -link_directories(${UNBOUND_LIBRARY_DIRS}) +include_directories(${UNBOUND_INCLUDE_DIR}) # Final setup for easylogging++ include_directories(${EASYLOGGING_INCLUDE}) @@ -793,14 +788,14 @@ else() endif() set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes") set(CXX_WARNINGS "-Wno-reorder -Wno-missing-field-initializers") - try_compile(STATIC_ASSERT_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.c" COMPILE_DEFINITIONS "-std=c11") + try_compile(STATIC_ASSERT_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.c" CMAKE_FLAGS -DCMAKE_CXX_STANDARD=11) if(STATIC_ASSERT_RES) set(STATIC_ASSERT_FLAG "") else() set(STATIC_ASSERT_FLAG "-Dstatic_assert=_Static_assert") endif() - try_compile(STATIC_ASSERT_CPP_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.cpp" COMPILE_DEFINITIONS "-std=c++11") + try_compile(STATIC_ASSERT_CPP_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.cpp" CMAKE_FLAGS -DCMAKE_CXX_STANDARD=11) if(STATIC_ASSERT_CPP_RES) set(STATIC_ASSERT_CPP_FLAG "") else() @@ -896,8 +891,8 @@ else() message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}") # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that @@ -1242,10 +1237,6 @@ endif() # when ON - will install libwallet_merged into "lib" option(BUILD_GUI_DEPS "Build GUI dependencies." OFF) -# This is not nice, distribution packagers should not enable this, but depend -# on libunbound shipped with their distribution instead -option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source vendored with this repo." OFF) - find_package(PythonInterp) find_program(iwyu_tool_path NAMES iwyu_tool.py iwyu_tool) if (iwyu_tool_path AND PYTHONINTERP_FOUND) diff --git a/Dockerfile b/Dockerfile index f21f74ac3..587007470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,192 +1,45 @@ # Multistage docker build, requires docker 17.05 # builder stage -FROM ubuntu:16.04 as builder +FROM ubuntu:20.04 as builder RUN set -ex && \ apt-get update && \ - apt-get --no-install-recommends --yes install \ + DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends --yes install \ + automake \ + autotools-dev \ + bsdmainutils \ + build-essential \ ca-certificates \ + ccache \ cmake \ - g++ \ - make \ - pkg-config \ - graphviz \ - doxygen \ - git \ curl \ - libtool-bin \ - autoconf \ - automake \ - bzip2 \ - xsltproc \ - gperf \ - unzip - -WORKDIR /usr/local - -ENV CFLAGS='-fPIC' -ENV CXXFLAGS='-fPIC' - -#Cmake -ARG CMAKE_VERSION=3.14.6 -ARG CMAKE_VERSION_DOT=v3.14 -ARG CMAKE_HASH=4e8ea11cabe459308671b476469eace1622e770317a15951d7b55a82ccaaccb9 -RUN set -ex \ - && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \ - && echo "${CMAKE_HASH} cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf cmake-${CMAKE_VERSION}.tar.gz \ - && cd cmake-${CMAKE_VERSION} \ - && ./configure \ - && make \ - && make install - -## Boost -ARG BOOST_VERSION=1_70_0 -ARG BOOST_VERSION_DOT=1.70.0 -ARG BOOST_HASH=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 -RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOT}/boost_${BOOST_VERSION}.tar.bz2 \ - && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ - && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ - && cd boost_${BOOST_VERSION} \ - && ./bootstrap.sh \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="$CFLAGS" cxxflags="$CXXFLAGS" stage -ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} - -# OpenSSL -ARG OPENSSL_VERSION=1.1.1i -ARG OPENSSL_HASH=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 -RUN set -ex \ - && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ - && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ - && cd openssl-${OPENSSL_VERSION} \ - && ./Configure linux-x86_64 no-shared --static "$CFLAGS" \ - && make build_generated \ - && make libcrypto.a \ - && make install -ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION} - -# ZMQ -ARG ZMQ_VERSION=v4.3.2 -ARG ZMQ_HASH=a84ffa12b2eb3569ced199660bac5ad128bff1f0 -RUN set -ex \ - && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ - && cd libzmq \ - && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure --enable-static --disable-shared \ - && make \ - && make install \ - && ldconfig - -# zmq.hpp -ARG CPPZMQ_VERSION=v4.4.1 -ARG CPPZMQ_HASH=f5b36e563598d48fcc0d82e589d3596afef945ae -RUN set -ex \ - && git clone https://github.com/zeromq/cppzmq.git -b ${CPPZMQ_VERSION} \ - && cd cppzmq \ - && test `git rev-parse HEAD` = ${CPPZMQ_HASH} || exit 1 \ - && mv *.hpp /usr/local/include - -# Readline -ARG READLINE_VERSION=8.0 -ARG READLINE_HASH=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 -RUN set -ex \ - && curl -s -O https://ftp.gnu.org/gnu/readline/readline-${READLINE_VERSION}.tar.gz \ - && echo "${READLINE_HASH} readline-${READLINE_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf readline-${READLINE_VERSION}.tar.gz \ - && cd readline-${READLINE_VERSION} \ - && ./configure \ - && make \ - && make install - -# Sodium -ARG SODIUM_VERSION=1.0.18 -ARG SODIUM_HASH=4f5e89fa84ce1d178a6765b8b46f2b6f91216677 -RUN set -ex \ - && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ - && cd libsodium \ - && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure \ - && make \ - && make check \ - && make install - -# Udev -ARG UDEV_VERSION=v3.2.8 -ARG UDEV_HASH=d69f3f28348123ab7fa0ebac63ec2fd16800c5e0 -RUN set -ex \ - && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ - && cd eudev \ - && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ - && make \ - && make install - -# Libusb -ARG USB_VERSION=v1.0.22 -ARG USB_HASH=0034b2afdcdb1614e78edaa2a9e22d5936aeae5d -RUN set -ex \ - && git clone https://github.com/libusb/libusb.git -b ${USB_VERSION} \ - && cd libusb \ - && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ - && ./autogen.sh \ - && ./configure --disable-shared \ - && make \ - && make install - -# Hidapi -ARG HIDAPI_VERSION=hidapi-0.8.0-rc1 -ARG HIDAPI_HASH=40cf516139b5b61e30d9403a48db23d8f915f52c -RUN set -ex \ - && git clone https://github.com/signal11/hidapi -b ${HIDAPI_VERSION} \ - && cd hidapi \ - && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ - && ./bootstrap \ - && ./configure --enable-static --disable-shared \ - && make \ - && make install - -# Protobuf -ARG PROTOBUF_VERSION=v3.7.1 -ARG PROTOBUF_HASH=6973c3a5041636c1d8dc5f7f6c8c1f3c15bc63d6 -RUN set -ex \ - && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ - && cd protobuf \ - && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \ - && git submodule update --init --recursive \ - && ./autogen.sh \ - && ./configure --enable-static --disable-shared \ - && make \ - && make install \ - && ldconfig + git \ + libtool \ + pkg-config \ + gperf WORKDIR /src COPY . . -ENV USE_SINGLE_BUILDDIR=1 ARG NPROC RUN set -ex && \ git submodule init && git submodule update && \ rm -rf build && \ if [ -z "$NPROC" ] ; \ - then make -j$(nproc) release-static ; \ - else make -j$NPROC release-static ; \ + then make -j$(nproc) depends target=x86_64-linux-gnu ; \ + else make -j$NPROC depends target=x86_64-linux-gnu ; \ fi # runtime stage -FROM ubuntu:16.04 +FROM ubuntu:20.04 RUN set -ex && \ apt-get update && \ apt-get --no-install-recommends --yes install ca-certificates && \ apt-get clean && \ rm -rf /var/lib/apt -COPY --from=builder /src/build/release/bin /usr/local/bin/ +COPY --from=builder /src/build/x86_64-linux-gnu/release/bin /usr/local/bin/ # Create monero user RUN adduser --system --group --disabled-password monero && \ @@ -123,12 +123,12 @@ release-static-linux-armv7: release-static-android-armv7: mkdir -p $(builddir)/release/translations cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE) - cd $(builddir)/release && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android-armv7" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARM_MODE=ON -D CMAKE_ANDROID_ARCH_ABI="armeabi-v7a" ../.. && $(MAKE) + cd $(builddir)/release && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-armv7" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARM_MODE=ON -D CMAKE_ANDROID_ARCH_ABI="armeabi-v7a" ../.. && $(MAKE) release-static-android-armv8: mkdir -p $(builddir)/release/translations cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE) - cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE) + cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE) release-static-linux-armv8: mkdir -p $(builddir)/release @@ -42,12 +42,9 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Research -The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. +The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with the Monero research community if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. -Our researchers are available on IRC in [#monero-research-lab on Libera](https://web.libera.chat/#monero-research-lab) or by email: - -- Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) -- Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) +The Monero research community is available on IRC in [#monero-research-lab on Libera](https://web.libera.chat/#monero-research-lab), which is also accessible via Matrix. ## Announcements @@ -202,7 +199,7 @@ Install all dependencies at once on macOS with the provided Brewfile: ``` brew update && brew bundle --file=contrib/brew/Brewfile ``` FreeBSD 12.1 one-liner required to build dependencies: -```pkg install git gmake cmake pkgconf boost-libs libzmq4 libsodium``` +```pkg install git gmake cmake pkgconf boost-libs libzmq4 libsodium unbound``` ### Cloning the repository @@ -399,13 +396,13 @@ application. To build for 64-bit Windows: ```bash - pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi + pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-unbound ``` To build for 32-bit Windows: ```bash - pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi + pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi mingw-w64-i686-unbound ``` * Open the MingW shell via `MinGW-w64-Win64 Shell` shortcut on 64-bit Windows diff --git a/contrib/depends/packages/expat.mk b/contrib/depends/packages/expat.mk index d73a5e307..1e1b9dbb8 100644 --- a/contrib/depends/packages/expat.mk +++ b/contrib/depends/packages/expat.mk @@ -1,8 +1,8 @@ package=expat -$(package)_version=2.2.4 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_4 +$(package)_version=2.4.1 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_4_1 $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd372019e +$(package)_sha256_hash=2f9b6a580b94577b150a7d5617ad4643a4301a6616ff459307df3e225bcfbf40 define $(package)_set_vars $(package)_config_opts=--enable-static diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index 6fbcc3466..90c63e821 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -1,8 +1,8 @@ package=ldns -$(package)_version=1.6.17 -$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/ +$(package)_version=1.7.1 +$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd +$(package)_sha256_hash=8ac84c16bdca60e710eea75782356f3ac3b55680d40e1530d7cea474ac208229 $(package)_dependencies=openssl define $(package)_set_vars diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index d80775b22..16c232b41 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -59,11 +59,11 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) -j1 build_libs libcrypto.pc libssl.pc openssl.pc + $(MAKE) build_libs endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw + $(MAKE) DESTDIR=$($(package)_staging_dir) install_sw endef define $(package)_postprocess_cmds diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 95b23a37e..56ce425bb 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,4 +1,4 @@ -packages:=boost openssl zeromq libiconv +packages:=boost openssl zeromq libiconv expat ldns unbound native_packages := native_ccache diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk index 733a7f232..2d870d63f 100644 --- a/contrib/depends/packages/unbound.mk +++ b/contrib/depends/packages/unbound.mk @@ -1,12 +1,12 @@ package=unbound -$(package)_version=1.6.8 -$(package)_download_path=https://www.unbound.net/downloads/ +$(package)_version=1.13.2 +$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49 +$(package)_sha256_hash=0a13b547f3b92a026b5ebd0423f54c991e5718037fd9f72445817f6a040e1a83 $(package)_dependencies=openssl expat ldns define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads + $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only $(package)_config_opts_linux=--with-pic $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix) $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread" diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk index 826a820c4..c3d190bca 100644 --- a/contrib/depends/packages/unwind.mk +++ b/contrib/depends/packages/unwind.mk @@ -1,8 +1,8 @@ package=unwind -$(package)_version=1.2 +$(package)_version=1.5.0 $(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind $(package)_file_name=lib$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992 +$(package)_sha256_hash=90337653d92d4a13de590781371c604f9031cdb50520366aa1e3a91e1efb1017 $(package)_patches=fix_obj_order.patch define $(package)_preprocess_cmds @@ -12,7 +12,7 @@ endef define $(package)_config_cmds cp -f $(BASEDIR)/config.guess config/config.guess &&\ cp -f $(BASEDIR)/config.sub config/config.sub &&\ - $($(package)_autoconf) --disable-shared --enable-static AR_FLAGS=$($(package)_arflags) + $($(package)_autoconf) --disable-shared --enable-static --disable-tests --disable-documentation AR_FLAGS=$($(package)_arflags) endef define $(package)_build_cmds diff --git a/contrib/depends/patches/unwind/fix_obj_order.patch b/contrib/depends/patches/unwind/fix_obj_order.patch index 374a9f04a..e764f0f3d 100644 --- a/contrib/depends/patches/unwind/fix_obj_order.patch +++ b/contrib/depends/patches/unwind/fix_obj_order.patch @@ -1,9 +1,9 @@ ---- config/ltmain.sh.O 2017-01-13 16:00:54.000000000 +0000 -+++ config/ltmain.sh 2019-11-17 06:46:51.994402494 +0000 -@@ -7957,6 +7957,8 @@ - esac - done +--- config/ltmain.sh.0 2020-11-10 17:25:26.000000000 +0100 ++++ config/ltmain.sh 2021-09-11 19:39:36.000000000 +0200 +@@ -10768,6 +10768,8 @@ fi + func_to_tool_file "$oldlib" func_convert_file_msys_to_w32 + tool_oldlib=$func_to_tool_file_result + oldobjs=`for obj in $oldobjs; do echo $obj; done | sort` + oldobjs=" `echo $oldobjs`" eval cmds=\"$old_archive_cmds\" diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index 383b88f31..a87b9c058 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -24,6 +24,9 @@ SET(Readline_INCLUDE_DIR @prefix@/include) SET(Readline_LIBRARY @prefix@/lib/libreadline.a) SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a) +SET(UNBOUND_INCLUDE_DIR @prefix@/include) +SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a) + SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index 19bdf4ff0..0f4a28c99 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -668,7 +668,7 @@ namespace net_utils // Cross-origin resource sharing if(m_query_info.m_header_info.m_origin.size()) { - if (std::binary_search(m_config.m_access_control_origins.begin(), m_config.m_access_control_origins.end(), m_query_info.m_header_info.m_origin)) + if (std::binary_search(m_config.m_access_control_origins.begin(), m_config.m_access_control_origins.end(), "*") || std::binary_search(m_config.m_access_control_origins.begin(), m_config.m_access_control_origins.end(), m_query_info.m_header_info.m_origin)) { buf += "Access-Control-Allow-Origin: "; buf += m_query_info.m_header_info.m_origin; diff --git a/contrib/epee/include/stats.inl b/contrib/epee/include/stats.inl index 5a5cd0b93..70c127be7 100644 --- a/contrib/epee/include/stats.inl +++ b/contrib/epee/include/stats.inl @@ -1,6 +1,7 @@ #include <math.h> #include <limits> #include <algorithm> +#include "misc_language.h" #include "stats.h" enum @@ -86,7 +87,7 @@ Tpod Stats<T, Tpod>::get_median() const } else { - median = (sorted[(sorted.size() - 1) / 2] + sorted[sorted.size() / 2]) / 2; + median = epee::misc_utils::get_mid(sorted[(sorted.size() - 1) / 2], sorted[sorted.size() / 2]); } set_cached(bit_median); } diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 07f119ba4..3fa2f7fb4 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -47,9 +47,7 @@ if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) monero_add_library(epee_readline readline_buffer.cpp) endif() -if(HAVE_C11) -SET_PROPERTY(SOURCE memwipe.c PROPERTY COMPILE_FLAGS -std=c11) -endif() +set_property(SOURCE memwipe.c PROPERTY C_STANDARD 11) # Build and install libepee if we're building for GUI if (BUILD_GUI_DEPS) diff --git a/contrib/epee/src/http_base.cpp b/contrib/epee/src/http_base.cpp index 647dfb899..f6d7568c5 100644 --- a/contrib/epee/src/http_base.cpp +++ b/contrib/epee/src/http_base.cpp @@ -44,7 +44,7 @@ namespace http std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) { fields_list::const_iterator it = fields.begin(); - for(; it != fields.end(); it++) + for(; it != fields.end(); ++it) if(!string_tools::compare_no_case(param_name, it->first)) break; diff --git a/contrib/epee/src/net_parse_helpers.cpp b/contrib/epee/src/net_parse_helpers.cpp index de7843b67..2697bdac4 100644 --- a/contrib/epee/src/net_parse_helpers.cpp +++ b/contrib/epee/src/net_parse_helpers.cpp @@ -48,7 +48,7 @@ namespace net_utils state st = st_param_name; std::string::const_iterator start_it = query.begin(); std::pair<std::string, std::string> e; - for(std::string::const_iterator it = query.begin(); it != query.end(); it++) + for(std::string::const_iterator it = query.begin(); it != query.end(); ++it) { switch(st) { diff --git a/contrib/epee/src/parserse_base_utils.cpp b/contrib/epee/src/parserse_base_utils.cpp index 112e9c5e4..e96c2dede 100644 --- a/contrib/epee/src/parserse_base_utils.cpp +++ b/contrib/epee/src/parserse_base_utils.cpp @@ -102,7 +102,7 @@ namespace misc_utils ++fi; val.assign(it, fi); it = fi; - for(;it != buf_end;it++) + for(;it != buf_end;++it) { if(escape_mode/*prev_ch == '\\'*/) { @@ -197,7 +197,7 @@ namespace misc_utils ++chars; ++it; } - for(;it != buf_end;it++) + for(;it != buf_end;++it) { const uint8_t flags = lut[(uint8_t)*it]; if (flags & 16) @@ -224,7 +224,7 @@ namespace misc_utils { val.clear(); - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + for(std::string::const_iterator it = star_end_string;it != buf_end;++it) { if (!(lut[(uint8_t)*it] & 4)) { @@ -243,7 +243,7 @@ namespace misc_utils { val.clear(); - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + for(std::string::const_iterator it = star_end_string;it != buf_end;++it) { if(!isalnum(*it) && *it != '-' && *it != '_') { @@ -262,7 +262,7 @@ namespace misc_utils { word_end = star_end_string; - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) + for(std::string::const_iterator it = star_end_string;it != buf_end;++it) { if(isspace(*it)) { diff --git a/contrib/epee/tests/src/CMakeLists.txt b/contrib/epee/tests/src/CMakeLists.txt index 0c42f87ca..e724b53f4 100644 --- a/contrib/epee/tests/src/CMakeLists.txt +++ b/contrib/epee/tests/src/CMakeLists.txt @@ -1,6 +1,13 @@ cmake_minimum_required(VERSION 3.5) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + set(Boost_USE_MULTITHREADED ON) include_directories(.) @@ -14,8 +21,8 @@ IF (MSVC) include_directories(SYSTEM platform/msvc) ELSE() # set stuff for other systems - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -Wall") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-reorder") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-reorder") ENDIF() diff --git a/docs/ZMQ.md b/docs/ZMQ.md index 9128ff2ad..a7006f486 100644 --- a/docs/ZMQ.md +++ b/docs/ZMQ.md @@ -25,6 +25,8 @@ allows for filtering on: (1) format, (2) context, and (3) event. Includes previously unseen transactions in a block but _not_ the `miner_tx`. Does not "re-publish" after a reorg. Includes `do_not_relay` transactions. + * `miner_data` - provides the necessary data to create a custom block template + Available only in the `full` context. The subscription topics are formatted as `format-context-event`, with prefix matching supported by both Monero and ZMQ. The `format`, `context` and `event` diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 7ae4ba750..5b7f69a56 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -55,26 +55,12 @@ set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE) find_package(Unbound) -if(NOT UNBOUND_INCLUDE_DIR OR STATIC) - # NOTE: If STATIC is true, CMAKE_FIND_LIBRARY_SUFFIXES has been reordered. - # unbound has config tests which used OpenSSL libraries, so -ldl may need to - # be set in this case. - # The unbound CMakeLists.txt can set it, since it's also needed for the - # static OpenSSL libraries set up there after with target_link_libraries. - add_subdirectory(unbound) - - set(UNBOUND_STATIC true PARENT_SCOPE) - set(UNBOUND_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/unbound/libunbound" PARENT_SCOPE) - set(UNBOUND_LIBRARY "unbound" PARENT_SCOPE) - set(UNBOUND_LIBRARY_DIRS "${LIBEVENT2_LIBDIR}" PARENT_SCOPE) +if(NOT UNBOUND_INCLUDE_DIR) + die("Could not find libunbound") else() message(STATUS "Found libunbound include (unbound.h) in ${UNBOUND_INCLUDE_DIR}") if(UNBOUND_LIBRARIES) - message(STATUS "Found libunbound shared library") - set(UNBOUND_STATIC false PARENT_SCOPE) - set(UNBOUND_INCLUDE ${UNBOUND_INCLUDE_DIR} PARENT_SCOPE) - set(UNBOUND_LIBRARY ${UNBOUND_LIBRARIES} PARENT_SCOPE) - set(UNBOUND_LIBRARY_DIRS "" PARENT_SCOPE) + message(STATUS "Found libunbound library") else() die("Found libunbound includes, but could not find libunbound library. Please make sure you have installed libunbound or libunbound-dev or the equivalent") endif() diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 9aa4c08bc..462f774bf 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -30,7 +30,10 @@ cmake_minimum_required(VERSION 3.5) project(easylogging CXX) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + monero_enable_coverage() find_package(Threads) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 267770074..a765ee8cc 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2984,8 +2984,8 @@ void Writer::initializeLogger(Logger *logger, bool needLock) { } void Writer::processDispatch() { - static std::atomic_flag in_dispatch; - if (in_dispatch.test_and_set()) + static __thread bool in_dispatch = false; + if (in_dispatch) { if (m_proceed && m_logger != NULL) { @@ -2994,6 +2994,7 @@ void Writer::processDispatch() { } return; } + in_dispatch = true; #if ELPP_LOGGING_ENABLED if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { bool firstDispatched = false; @@ -3032,7 +3033,7 @@ void Writer::processDispatch() { m_logger->releaseLock(); } #endif // ELPP_LOGGING_ENABLED - in_dispatch.clear(); + in_dispatch = false; } void Writer::triggerDispatch(void) { diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index c4a88339f..b983a796c 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -2026,6 +2026,7 @@ class TypedConfigurations : public base::threading::ThreadSafe { ELPP_INTERNAL_ERROR("Unable to get configuration [" << confName << "] for level [" << LevelHelper::convertToString(level) << "]" << std::endl << "Please ensure you have properly configured logger.", false); + throw; // The exception has to be rethrown, to abort a branch leading to UB. } } return it->second; diff --git a/external/unbound b/external/unbound deleted file mode 160000 -Subproject 0f6c0579d66b65f86066e30e7876105ba2775ef diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 8e427b6b8..99d9bd8bf 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -99,7 +99,7 @@ monero_add_library(common target_link_libraries(common PUBLIC cncrypto - ${UNBOUND_LIBRARY} + ${UNBOUND_LIBRARIES} ${LIBUNWIND_LIBRARIES} ${Boost_DATE_TIME_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 6394a7071..d111f6f32 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -49,6 +49,7 @@ #include "misc_language.h" #include "ringct/rctTypes.h" #include "device/device.hpp" +#include "cryptonote_basic/fwd.h" namespace cryptonote { diff --git a/src/cryptonote_basic/events.h b/src/cryptonote_basic/events.h index 6c6742215..3417ece8c 100644 --- a/src/cryptonote_basic/events.h +++ b/src/cryptonote_basic/events.h @@ -41,6 +41,8 @@ namespace cryptonote { cryptonote::transaction tx; crypto::hash hash; + uint64_t blob_size; + uint64_t weight; bool res; //!< Listeners must ignore `tx` when this is false. }; } diff --git a/src/cryptonote_basic/fwd.h b/src/cryptonote_basic/fwd.h index d54223461..901ad151b 100644 --- a/src/cryptonote_basic/fwd.h +++ b/src/cryptonote_basic/fwd.h @@ -33,4 +33,5 @@ namespace cryptonote struct block; class transaction; struct txpool_event; + struct tx_block_template_backlog_entry; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index f0e6794b9..3d0e81af1 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1238,6 +1238,12 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + crypto::hash prev_id; + if (!get_block_hash(alt_chain.back().bl, prev_id)) + MERROR("Failed to get block hash of an alternative chain's tip"); + else + send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + for (const auto& notifier : m_block_notifiers) { std::size_t notify_height = split_height; @@ -1784,6 +1790,30 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //------------------------------------------------------------------ +bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + prev_id = m_db->top_block_hash(&height); + ++height; + + major_version = m_hardfork->get_ideal_version(height); + + seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + difficulty = get_difficulty_for_next_block(); + median_weight = m_current_block_cumul_weight_median; + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + + m_tx_pool.get_block_template_backlog(tx_backlog); + + return true; +} +//------------------------------------------------------------------ // for an alternate chain, get the timestamps from the main chain to complete // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const @@ -4362,6 +4392,7 @@ leave: get_difficulty_for_next_block(); // just to cache it invalidate_block_template_cache(); + send_miner_notifications(id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); @@ -5270,7 +5301,7 @@ void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint m_max_prepare_blocks_threads = maxthreads; } -void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)>&& notify) +void Blockchain::add_block_notify(BlockNotifyCallback&& notify) { if (notify) { @@ -5279,6 +5310,15 @@ void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span } } +void Blockchain::add_miner_notify(MinerNotifyCallback&& notify) +{ + if (notify) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_miner_notifiers.push_back(std::move(notify)); + } +} + void Blockchain::safesyncmode(const bool onoff) { /* all of this is no-op'd if the user set a specific @@ -5531,6 +5571,33 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_valid = true; } +void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins) +{ + if (m_miner_notifiers.empty()) + return; + + const uint64_t height = m_db->height(); + const uint8_t major_version = m_hardfork->get_ideal_version(height); + const difficulty_type diff = get_difficulty_for_next_block(); + const uint64_t median_weight = m_current_block_cumul_weight_median; + + crypto::hash seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + std::vector<tx_block_template_backlog_entry> tx_backlog; + m_tx_pool.get_block_template_backlog(tx_backlog); + + for (const auto& notifier : m_miner_notifiers) + { + notifier(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + } +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index a0e7967de..9afbfbc2d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -90,6 +90,9 @@ namespace cryptonote */ typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback; + typedef boost::function<void(uint64_t /* height */, epee::span<const block> /* blocks */)> BlockNotifyCallback; + typedef boost::function<void(uint8_t /* major_version */, uint64_t /* height */, const crypto::hash& /* prev_id */, const crypto::hash& /* seed_hash */, difficulty_type /* diff */, uint64_t /* median_weight */, uint64_t /* already_generated_coins */, const std::vector<tx_block_template_backlog_entry>& /* tx_backlog */)> MinerNotifyCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -371,6 +374,22 @@ namespace cryptonote bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @brief gets data required to create a block template and start mining on it + * + * @param major_version current hardfork version + * @param height current blockchain height + * @param prev_id hash of the top block + * @param seed_hash seed hash used for RandomX initialization + * @param difficulty current mining difficulty + * @param median_weight current median block weight + * @param already_generated_coins current emission + * @param tx_backlog transactions in mempool ready to be mined + * + * @return true if block template filled in successfully, else false + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief checks if a block is known about with a given hash * * This function checks the main chain, alternate chains, and invalid blocks @@ -775,7 +794,14 @@ namespace cryptonote * * @param notify the notify object to call at every new block */ - void add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)> &¬ify); + void add_block_notify(BlockNotifyCallback&& notify); + + /** + * @brief sets a miner notify object to call for every new block + * + * @param notify the notify object to call at every new block + */ + void add_miner_notify(MinerNotifyCallback&& notify); /** * @brief sets a reorg notify object to call for every reorg @@ -1157,7 +1183,8 @@ namespace cryptonote the callable object has a single `std::shared_ptr` or `std::weap_ptr` internally. Whereas, the libstdc++ `std::function` will allocate. */ - std::vector<boost::function<void(std::uint64_t, epee::span<const block>)>> m_block_notifiers; + std::vector<BlockNotifyCallback> m_block_notifiers; + std::vector<MinerNotifyCallback> m_miner_notifiers; std::shared_ptr<tools::Notify> m_reorg_notify; // for prepare_handle_incoming_blocks @@ -1537,5 +1564,13 @@ namespace cryptonote * At some point, may be used to push an update to miners */ void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t seed_height, const crypto::hash &seed_hash, uint64_t pool_cookie); + + /** + * @brief sends new block notifications to ZMQ `miner_data` subscribers + * + * @param prev_id hash of new blockchain tip + * @param already_generated_coins total coins mined by the network so far + */ + void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 1da14221a..4c6536318 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -388,6 +388,7 @@ namespace cryptonote m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); + if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) MWARNING(arg_fluffy_blocks.name << " is obsolete, it is now default"); @@ -460,7 +461,7 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */) + bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */, bool allow_dns) { start_time = std::time(nullptr); @@ -471,6 +472,7 @@ namespace cryptonote } bool r = handle_command_line(vm); CHECK_AND_ASSERT_MES(r, false, "Failed to handle command line"); + m_disable_dns_checkpoints |= not allow_dns; std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); bool db_salvage = command_line::get_arg(vm, cryptonote::arg_db_salvage) != 0; @@ -697,7 +699,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(update_checkpoints(skip_dns_checkpoints), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); // DNS versions checking - if (check_updates_string == "disabled") + if (check_updates_string == "disabled" || not allow_dns) check_updates_level = UPDATES_DISABLED; else if (check_updates_string == "notify") check_updates_level = UPDATES_NOTIFY; @@ -1063,8 +1065,9 @@ namespace cryptonote if (already_have[i]) continue; - const uint64_t weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], tx_relay, relayed); + results[i].blob_size = it->blob.size(); + results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, results[i].weight, tvc[i], tx_relay, relayed); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} @@ -1403,6 +1406,11 @@ namespace cryptonote return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //----------------------------------------------------------------------------------------------- + bool core::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) + { + return m_blockchain_storage.get_miner_data(major_version, height, prev_id, seed_hash, difficulty, median_weight, already_generated_coins, tx_backlog); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8478049f9..d2bffdaee 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -237,6 +237,13 @@ namespace cryptonote virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @copydoc Blockchain::get_miner_data + * + * @note see Blockchain::get_miner_data + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief called when a transaction is relayed. * @note Should only be invoked from `levin_notify`. */ @@ -276,10 +283,11 @@ namespace cryptonote * @param vm command line parameters * @param test_options configuration options for testing * @param get_checkpoints if set, will be called to get checkpoints data, must return checkpoints data pointer and size or nullptr if there ain't any checkpoints for specific network type + * @param allow_dns whether or not to allow DNS requests * * @return false if one of the init steps fails, otherwise true */ - bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr); + bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr, bool allow_dns = true); /** * @copydoc Blockchain::reset_and_set_genesis_block diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 73cdd31cd..06412d6bf 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -108,6 +108,15 @@ namespace cryptonote }; //--------------------------------------------------------------- + + struct tx_block_template_backlog_entry + { + crypto::hash id; + uint64_t weight; + uint64_t fee; + }; + + //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index a7e96e23a..84605d6f5 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -913,6 +913,32 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + void tx_memory_pool::get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); + txpool_tx_meta_t tmp_meta; + m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ + transaction tx; + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) + { + MERROR("Failed to parse tx from txpool"); + // continue + return true; + } + tx.set_hash(txid); + + tmp_meta = meta; + + if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx)) + backlog.push_back({txid, meta.weight, meta.fee}); + + return true; + }, true, category); + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1222,11 +1248,11 @@ namespace cryptonote return ret; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref& txblob, transaction &tx) const { - struct transction_parser + struct transaction_parser { - transction_parser(const cryptonote::blobdata &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} + transaction_parser(const cryptonote::blobdata_ref &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} cryptonote::transaction &operator()() { if (!parsed) @@ -1238,7 +1264,7 @@ namespace cryptonote } return tx; } - const cryptonote::blobdata &txblob; + const cryptonote::blobdata_ref &txblob; const crypto::hash &txid; transaction &tx; bool parsed; @@ -1289,6 +1315,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata& txblob, transaction &tx) const + { + return is_transaction_ready_to_go(txd, txid, cryptonote::blobdata_ref{txblob.data(), txblob.size()}, tx); + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::have_key_images(const std::unordered_set<crypto::key_image>& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index ab2a57ea2..80b38c51d 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -266,6 +266,15 @@ namespace cryptonote void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive = false) const; /** + * @brief get (hash, weight, fee) for all transactions in the pool - the minimum required information to create a block template + * + * @param backlog return-by-reference that data + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes + * + */ + void get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive = false) const; + + /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics @@ -540,6 +549,7 @@ namespace cryptonote * * @return true if the transaction is good to go, otherwise false */ + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref &txblob, transaction&tx) const; bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction&tx) const; /** diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 6c3e163e6..a988fe25f 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -96,6 +96,16 @@ namespace daemon_args , 0 }; + const command_line::arg_descriptor<std::string> arg_proxy = { + "proxy", + "Network communication through proxy: <socks-ip:port> i.e. \"127.0.0.1:9050\"", + "", + }; + const command_line::arg_descriptor<bool> arg_proxy_allow_dns_leaks = { + "proxy-allow-dns-leaks", + "Allow DNS leaks outside of proxy", + false, + }; const command_line::arg_descriptor<bool> arg_public_node = { "public-node" , "Allow other users to use the node as a remote (restricted RPC mode, view-only commands) and advertise it over P2P" diff --git a/src/daemon/core.h b/src/daemon/core.h index 804d7474d..0811cf420 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -32,6 +32,7 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "misc_log_ex.h" +#include "daemon/command_line_args.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -66,7 +67,14 @@ public: #else const cryptonote::GetCheckpointsCallback& get_checkpoints = nullptr; #endif - if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints)) + + if (command_line::is_arg_defaulted(vm, daemon_args::arg_proxy) && command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks)) { + MLOG_RED(el::Level::Warning, "--" << daemon_args::arg_proxy_allow_dns_leaks.name << " is enabled, but --" + << daemon_args::arg_proxy.name << " is not specified."); + } + + const bool allow_dns = command_line::is_arg_defaulted(vm, daemon_args::arg_proxy) || command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks); + if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints, allow_dns)) { throw std::runtime_error("Failed to initialize core"); } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 99430b2b0..3f1885423 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -120,6 +120,7 @@ public: if (shared) { core.get().get_blockchain_storage().add_block_notify(cryptonote::listener::zmq_pub::chain_main{shared}); + core.get().get_blockchain_storage().add_miner_notify(cryptonote::listener::zmq_pub::miner_data{shared}); core.get().set_txpool_listener(cryptonote::listener::zmq_pub::txpool_add{shared}); } } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index d413906df..70aec5538 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -152,6 +152,8 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); + command_line::add_arg(core_settings, daemon_args::arg_proxy); + command_line::add_arg(core_settings, daemon_args::arg_proxy_allow_dns_leaks); command_line::add_arg(core_settings, daemon_args::arg_public_node); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index f68efccc2..38862c017 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -33,6 +33,7 @@ #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "p2p/net_node.h" #include "daemon/protocol.h" +#include "daemon/command_line_args.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -61,7 +62,7 @@ public: { //initialize objects MGINFO("Initializing p2p server..."); - if (!m_server.init(vm)) + if (!m_server.init(vm, command_line::get_arg(vm, daemon_args::arg_proxy), command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks))) { throw std::runtime_error("Failed to initialize p2p server."); } diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index af48bcc45..bff7dc449 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -62,7 +62,7 @@ public: { MGINFO("Initializing " << m_description << " RPC server..."); - if (!m_server.init(vm, restricted, port, allow_rpc_payment)) + if (!m_server.init(vm, restricted, port, allow_rpc_payment, command_line::get_arg(vm, daemon_args::arg_proxy))) { throw std::runtime_error("Failed to initialize " + m_description + " RPC server."); } diff --git a/src/device/device.hpp b/src/device/device.hpp index 582eb2242..6005e157d 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -91,7 +91,6 @@ namespace hw { public: device(): mode(NONE) {} - device(const device &hwdev) {} virtual ~device() {} explicit virtual operator bool() const = 0; diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index d435b448c..07009b9d2 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -162,6 +162,26 @@ namespace hw { * Live refresh process termination */ virtual void live_refresh_finish() =0; + + /** + * Requests public address, uses empty passphrase if asked for. + */ + virtual bool get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) =0; + + /** + * Reset session ID, restart with a new session. + */ + virtual void reset_session() =0; + + /** + * Returns true if device already asked for passphrase entry before (i.e., obviously supports passphrase entry) + */ + virtual bool seen_passphrase_entry_prompt() =0; + + /** + * Uses empty passphrase for all passphrase queries. + */ + virtual void set_use_empty_passphrase(bool always_use_empty_passphrase) =0; }; } diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index c2070b0d1..03e8bbba4 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -178,6 +178,15 @@ namespace trezor { } } + bool device_trezor::get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) { + m_reply_with_empty_passphrase = true; + const auto empty_passphrase_reverter = epee::misc_utils::create_scope_leave_handler([&]() { + m_reply_with_empty_passphrase = false; + }); + + return get_public_address(pubkey); + } + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { try { MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); @@ -206,6 +215,18 @@ namespace trezor { get_address(index, payment_id, true); } + void device_trezor::reset_session() { + m_device_session_id.clear(); + } + + bool device_trezor::seen_passphrase_entry_prompt() { + return m_seen_passphrase_entry_message; + } + + void device_trezor::set_use_empty_passphrase(bool always_use_empty_passphrase) { + m_always_use_empty_passphrase = always_use_empty_passphrase; + } + /* ======================================================================= */ /* Helpers */ /* ======================================================================= */ diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index d91d1de3f..15337d2b4 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -205,6 +205,26 @@ namespace trezor { const ::tools::wallet2::unsigned_tx_set & unsigned_tx, ::tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) override; + + /** + * Requests public address, uses empty passphrase if asked for. + */ + bool get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) override; + + /** + * Reset session ID, restart with a new session. + */ + virtual void reset_session() override; + + /** + * Returns true if device already asked for passphrase entry before (i.e., obviously supports passphrase entry) + */ + bool seen_passphrase_entry_prompt() override; + + /** + * Uses empty passphrase for all passphrase queries. + */ + void set_use_empty_passphrase(bool use_always_empty_passphrase) override; }; #endif diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index b0b4342f5..016eb2816 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -45,7 +45,10 @@ namespace trezor { const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; - device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success) { + device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success), + m_reply_with_empty_passphrase(false), + m_always_use_empty_passphrase(false), + m_seen_passphrase_entry_message(false) { #ifdef WITH_TREZOR_DEBUGGING m_debug = false; #endif @@ -155,6 +158,9 @@ namespace trezor { TREZOR_AUTO_LOCK_DEVICE(); m_device_session_id.clear(); m_features.reset(); + m_seen_passphrase_entry_message = false; + m_reply_with_empty_passphrase = false; + m_always_use_empty_passphrase = false; if (m_transport){ try { @@ -476,6 +482,7 @@ namespace trezor { return; } + m_seen_passphrase_entry_message = true; bool on_device = true; if (msg->has__on_device() && !msg->_on_device()){ on_device = false; // do not enter on device, old devices. @@ -491,19 +498,21 @@ namespace trezor { } boost::optional<epee::wipeable_string> passphrase; - TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + if (m_reply_with_empty_passphrase || m_always_use_empty_passphrase) { + MDEBUG("Answering passphrase prompt with an empty passphrase, always use empty: " << m_always_use_empty_passphrase); + on_device = false; + passphrase = epee::wipeable_string(""); + } else if (m_passphrase){ + MWARNING("Answering passphrase prompt with a stored passphrase (do not use; passphrase can be seen by a potential malware / attacker)"); + on_device = false; + passphrase = epee::wipeable_string(m_passphrase.get()); + } else { + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + } messages::common::PassphraseAck m; m.set_on_device(on_device); if (!on_device) { - if (!passphrase && m_passphrase) { - passphrase = m_passphrase; - } - - if (m_passphrase) { - m_passphrase = boost::none; - } - if (passphrase) { m.set_allocated_passphrase(new std::string(passphrase->data(), passphrase->size())); } diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 0162b23df..de49397d5 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -101,6 +101,9 @@ namespace trezor { messages::MessageType m_last_msg_type; cryptonote::network_type network_type; + bool m_reply_with_empty_passphrase; + bool m_always_use_empty_passphrase; + bool m_seen_passphrase_entry_message; #ifdef WITH_TREZOR_DEBUGGING std::shared_ptr<trezor_debug_callback> m_debug_callback; diff --git a/src/net/socks.h b/src/net/socks.h index 739c972ab..506b53195 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -201,6 +201,13 @@ namespace socks std::shared_ptr<client> self_; void operator()(boost::system::error_code error = boost::system::error_code{}); }; + + //! Calls `async_close` on `self` at destruction. NOP if `nullptr`. + struct close_on_exit + { + std::shared_ptr<client> self; + ~close_on_exit() { async_close{std::move(self)}(); } + }; }; template<typename Handler> diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 84cc1581e..d9050200a 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -94,6 +94,9 @@ namespace case net::i2p_address::get_type_id(): set = client->set_connect_command(remote.as<net::i2p_address>()); break; + case epee::net_utils::ipv4_network_address::get_type_id(): + set = client->set_connect_command(remote.as<epee::net_utils::ipv4_network_address>()); + break; default: MERROR("Unsupported network address in socks_connect"); return false; @@ -339,6 +342,7 @@ namespace nodetool } }; + net::socks::client::close_on_exit close_client{}; boost::unique_future<client_result> socks_result{}; { boost::promise<client_result> socks_promise{}; @@ -347,6 +351,7 @@ namespace nodetool auto client = net::socks::make_connect_client( boost::asio::ip::tcp::socket{service}, net::socks::version::v4a, notify{std::move(socks_promise)} ); + close_client.self = client; if (!start_socks(std::move(client), proxy, remote)) return boost::none; } @@ -368,7 +373,10 @@ namespace nodetool { auto result = socks_result.get(); if (!result.first) + { + close_client.self.reset(); return {std::move(result.second)}; + } MERROR("Failed to make socks connection to " << remote.str() << " (via " << proxy << "): " << result.first.message()); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 35e775163..ac815a100 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -255,6 +255,7 @@ namespace nodetool m_offline(false), is_closing(false), m_network_id(), + m_enable_dns_seed_nodes(true), max_connections(1) {} virtual ~node_server(); @@ -263,7 +264,7 @@ namespace nodetool bool run(); network_zone& add_zone(epee::net_utils::zone zone); - bool init(const boost::program_options::variables_map& vm); + bool init(const boost::program_options::variables_map& vm, const std::string& proxy = {}, bool proxy_dns_leaks_allowed = {}); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listening_port;} @@ -512,6 +513,7 @@ namespace nodetool epee::net_utils::ssl_support_t m_ssl_support; + bool m_enable_dns_seed_nodes; bool m_enable_dns_blocklist; uint32_t max_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index ab30d0074..d4b39869c 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -741,6 +741,12 @@ namespace nodetool { return get_ip_seed_nodes(); } + if (!m_enable_dns_seed_nodes) + { + // TODO: a domain can be set through socks, so that the remote side does the lookup for the DNS seed nodes. + m_fallback_seed_nodes_added.test_and_set(); + return get_ip_seed_nodes(); + } std::set<std::string> full_addrs; @@ -880,10 +886,21 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) + bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm, const std::string& proxy, bool proxy_dns_leaks_allowed) { bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); + if (proxy.size()) + { + const auto endpoint = net::get_tcp_endpoint(proxy); + CHECK_AND_ASSERT_MES(endpoint, false, "Failed to parse proxy: " << proxy << " - " << endpoint.error()); + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + public_zone.m_connect = &socks_connect; + public_zone.m_proxy_address = *endpoint; + public_zone.m_can_pingback = false; + m_enable_dns_seed_nodes &= proxy_dns_leaks_allowed; + m_enable_dns_blocklist &= proxy_dns_leaks_allowed; + } if (m_nettype == cryptonote::TESTNET) { diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 620f7d0dd..784c90a4e 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -449,7 +449,6 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str STEP = STEP ? STEP : 192; MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); - static constexpr unsigned int mask = (1<<STRAUS_C)-1; std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; ge_cached cached; ge_p1p1 p1; @@ -483,6 +482,7 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str memcpy(bytes33, data[j].scalar.bytes, 32); bytes33[32] = 0; bytes = bytes33; + static constexpr unsigned int mask = (1<<STRAUS_C)-1; for (size_t i = 0; i < 256; ++i) digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; #else diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 8d8a68efb..942bfce0a 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -242,11 +242,11 @@ namespace cryptonote auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; - m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, proxy)); + m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, m_bootstrap_daemon_proxy.empty() ? proxy : m_bootstrap_daemon_proxy)); } else { - m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, proxy)); + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, m_bootstrap_daemon_proxy.empty() ? proxy : m_bootstrap_daemon_proxy)); } m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; @@ -264,8 +264,10 @@ namespace cryptonote , const bool restricted , const std::string& port , bool allow_rpc_payment + , const std::string& proxy ) { + m_bootstrap_daemon_proxy = proxy; m_restricted = restricted; m_net_server.set_threads_prefix("RPC"); m_net_server.set_connection_filter(&m_p2p); @@ -1861,6 +1863,43 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + if(!check_core_ready()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy"; + return false; + } + + crypto::hash prev_id, seed_hash; + difficulty_type difficulty; + + std::vector<tx_block_template_backlog_entry> tx_backlog; + if (!m_core.get_miner_data(res.major_version, res.height, prev_id, seed_hash, difficulty, res.median_weight, res.already_generated_coins, tx_backlog)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: failed to get miner data"; + LOG_ERROR("Failed to get miner data"); + return false; + } + + res.tx_backlog.clear(); + res.tx_backlog.reserve(tx_backlog.size()); + + for (const auto& entry : tx_backlog) + { + res.tx_backlog.emplace_back(COMMAND_RPC_GETMINERDATA::response::tx_backlog_entry{string_tools::pod_to_hex(entry.id), entry.weight, entry.fee}); + } + + res.prev_id = string_tools::pod_to_hex(prev_id); + res.seed_hash = string_tools::pod_to_hex(seed_hash); + res.difficulty = cryptonote::hex(difficulty); + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(add_aux_pow); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index b21e43ab0..84b14383a 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -91,7 +91,8 @@ namespace cryptonote const boost::program_options::variables_map& vm, const bool restricted, const std::string& port, - bool allow_rpc_payment + bool allow_rpc_payment, + const std::string& proxy = {} ); network_type nettype() const { return m_core.get_nettype(); } @@ -147,6 +148,7 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("get_miner_data", on_getminerdata, COMMAND_RPC_GETMINERDATA) MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) @@ -228,6 +230,7 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); @@ -289,6 +292,7 @@ private: nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_p2p; boost::shared_mutex m_bootstrap_daemon_mutex; std::unique_ptr<bootstrap_daemon> m_bootstrap_daemon; + std::string m_bootstrap_daemon_proxy; bool m_should_use_bootstrap_daemon; std::chrono::system_clock::time_point m_bootstrap_height_check_time; bool m_was_bootstrap_ever_used; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index ff8c98b98..1dbfc83a7 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 7 +#define CORE_RPC_VERSION_MINOR 8 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -940,6 +940,56 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_GETMINERDATA + { + struct request_t: public rpc_request_base + { + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + uint8_t major_version; + uint64_t height; + std::string prev_id; + std::string seed_hash; + std::string difficulty; + uint64_t median_weight; + uint64_t already_generated_coins; + + struct tx_backlog_entry + { + std::string id; + uint64_t weight; + uint64_t fee; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(weight) + KV_SERIALIZE(fee) + END_KV_SERIALIZE_MAP() + }; + + std::vector<tx_backlog_entry> tx_backlog; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(major_version) + KV_SERIALIZE(height) + KV_SERIALIZE(prev_id) + KV_SERIALIZE(seed_hash) + KV_SERIALIZE(difficulty) + KV_SERIALIZE(median_weight) + KV_SERIALIZE(already_generated_coins) + KV_SERIALIZE(tx_backlog) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_ADD_AUX_POW { struct aux_pow_t diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp index eac530968..074b55207 100644 --- a/src/rpc/zmq_pub.cpp +++ b/src/rpc/zmq_pub.cpp @@ -48,6 +48,8 @@ #include "cryptonote_basic/events.h" #include "misc_log_ex.h" #include "serialization/json_object.h" +#include "ringct/rctTypes.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.zmq" @@ -57,6 +59,7 @@ namespace constexpr const char txpool_signal[] = "tx_signal"; using chain_writer = void(epee::byte_stream&, std::uint64_t, epee::span<const cryptonote::block>); + using miner_writer = void(epee::byte_stream&, uint8_t, uint64_t, const crypto::hash&, const crypto::hash&, cryptonote::difficulty_type, uint64_t, uint64_t, const std::vector<cryptonote::tx_block_template_backlog_entry>&); using txpool_writer = void(epee::byte_stream&, epee::span<const cryptonote::txpool_event>); template<typename F> @@ -116,13 +119,30 @@ namespace const epee::span<const cryptonote::block> blocks; }; + //! Object for miner data serialization + struct miner_data + { + uint8_t major_version; + uint64_t height; + const crypto::hash& prev_id; + const crypto::hash& seed_hash; + cryptonote::difficulty_type diff; + uint64_t median_weight; + uint64_t already_generated_coins; + const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog; + }; + //! Object for "minimal" tx serialization struct minimal_txpool { const cryptonote::transaction& tx; + crypto::hash hash; + uint64_t blob_size; + uint64_t weight; + uint64_t fee; }; - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain& self) { namespace adapt = boost::adaptors; @@ -143,19 +163,27 @@ namespace dest.EndObject(); } - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const miner_data& self) { - crypto::hash id{}; - std::size_t blob_size = 0; - if (!get_transaction_hash(self.tx, id, blob_size)) - { - MERROR("ZMQ/Pub failure: get_transaction_hash"); - return; - } + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, major_version, self.major_version); + INSERT_INTO_JSON_OBJECT(dest, height, self.height); + INSERT_INTO_JSON_OBJECT(dest, prev_id, self.prev_id); + INSERT_INTO_JSON_OBJECT(dest, seed_hash, self.seed_hash); + INSERT_INTO_JSON_OBJECT(dest, difficulty, cryptonote::hex(self.diff)); + INSERT_INTO_JSON_OBJECT(dest, median_weight, self.median_weight); + INSERT_INTO_JSON_OBJECT(dest, already_generated_coins, self.already_generated_coins); + INSERT_INTO_JSON_OBJECT(dest, tx_backlog, self.tx_backlog); + dest.EndObject(); + } + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool& self) + { dest.StartObject(); - INSERT_INTO_JSON_OBJECT(dest, id, id); - INSERT_INTO_JSON_OBJECT(dest, blob_size, blob_size); + INSERT_INTO_JSON_OBJECT(dest, id, self.hash); + INSERT_INTO_JSON_OBJECT(dest, blob_size, self.blob_size); + INSERT_INTO_JSON_OBJECT(dest, weight, self.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, self.fee); dest.EndObject(); } @@ -169,6 +197,11 @@ namespace json_pub(buf, minimal_chain{height, blocks}); } + void json_miner_data(epee::byte_stream& buf, uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, cryptonote::difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog) + { + json_pub(buf, miner_data{major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog}); + } + // boost::adaptors are in place "views" - no copy/move takes place // moving transactions (via sort, etc.), is expensive! @@ -187,7 +220,7 @@ namespace namespace adapt = boost::adaptors; const auto to_minimal_tx = [](const cryptonote::txpool_event& event) { - return minimal_txpool{event.tx}; + return minimal_txpool{event.tx, event.hash, event.blob_size, event.weight, cryptonote::get_tx_fee(event.tx)}; }; json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_minimal_tx))); } @@ -198,6 +231,11 @@ namespace {u8"json-minimal-chain_main", json_minimal_chain} }}; + constexpr const std::array<context<miner_writer>, 1> miner_contexts = + {{ + {u8"json-full-miner_data", json_miner_data}, + }}; + constexpr const std::array<context<txpool_writer>, 2> txpool_contexts = {{ {u8"json-full-txpool_add", json_full_txpool}, @@ -321,6 +359,7 @@ namespace cryptonote { namespace listener zmq_pub::zmq_pub(void* context) : relay_(), chain_subs_{{0}}, + miner_subs_{{0}}, txpool_subs_{{0}}, sync_() { @@ -328,6 +367,7 @@ zmq_pub::zmq_pub(void* context) throw std::logic_error{"ZMQ context cannot be NULL"}; verify_sorted(chain_contexts, "chain_contexts"); + verify_sorted(miner_contexts, "miner_contexts"); verify_sorted(txpool_contexts, "txpool_contexts"); relay_.reset(zmq_socket(context, ZMQ_PAIR)); @@ -348,22 +388,25 @@ bool zmq_pub::sub_request(boost::string_ref message) message.remove_prefix(1); const auto chain_range = get_range(chain_contexts, message); + const auto miner_range = get_range(miner_contexts, message); const auto txpool_range = get_range(txpool_contexts, message); - if (!chain_range.empty() || !txpool_range.empty()) + if (!chain_range.empty() || !miner_range.empty() || !txpool_range.empty()) { MDEBUG("Client " << (tag ? "subscribed" : "unsubscribed") << " to " << - chain_range.size() << " chain topic(s) and " << txpool_range.size() << " txpool topic(s)"); + chain_range.size() << " chain topic(s), " << miner_range.size() << " miner topic(s) and " << txpool_range.size() << " txpool topic(s)"); const boost::lock_guard<boost::mutex> lock{sync_}; switch (tag) { case 0: remove_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + remove_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); remove_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; case 1: add_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + add_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); add_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; default: @@ -436,6 +479,25 @@ std::size_t zmq_pub::send_chain_main(const std::uint64_t height, const epee::spa return 0; } +std::size_t zmq_pub::send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + boost::unique_lock<boost::mutex> guard{sync_}; + + const auto subs_copy = miner_subs_; + guard.unlock(); + + for (const std::size_t sub : subs_copy) + { + if (sub) + { + auto messages = make_pubs(subs_copy, miner_contexts, major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + guard.lock(); + return send_messages(relay_.get(), messages); + } + } + return 0; +} + std::size_t zmq_pub::send_txpool_add(std::vector<txpool_event> txes) { if (txes.empty()) @@ -466,6 +528,15 @@ void zmq_pub::chain_main::operator()(const std::uint64_t height, epee::span<cons MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); } +void zmq_pub::miner_data::operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const +{ + const std::shared_ptr<zmq_pub> self = self_.lock(); + if (self) + self->send_miner_data(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + else + MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); +} + void zmq_pub::txpool_add::operator()(std::vector<cryptonote::txpool_event> txes) const { const std::shared_ptr<zmq_pub> self = self_.lock(); diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h index 02e6b8103..c636e1d7b 100644 --- a/src/rpc/zmq_pub.h +++ b/src/rpc/zmq_pub.h @@ -39,6 +39,7 @@ #include "cryptonote_basic/fwd.h" #include "net/zmq.h" #include "span.h" +#include "cryptonote_basic/difficulty.h" namespace cryptonote { namespace listener { @@ -59,6 +60,7 @@ class zmq_pub net::zmq::socket relay_; std::deque<std::vector<txpool_event>> txes_; std::array<std::size_t, 2> chain_subs_; + std::array<std::size_t, 1> miner_subs_; std::array<std::size_t, 2> txpool_subs_; boost::mutex sync_; //!< Synchronizes counts in `*_subs_` arrays. @@ -88,6 +90,11 @@ class zmq_pub \return Number of ZMQ messages sent to relay. */ std::size_t send_chain_main(std::uint64_t height, epee::span<const cryptonote::block> blocks); + /*! Send a `ZMQ_PUB` notification for a new miner data. + Thread-safe. + \return Number of ZMQ messages sent to relay. */ + std::size_t send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog); + /*! Send a `ZMQ_PUB` notification for new tx(es) being added to the local pool. Thread-safe. \return Number of ZMQ messages sent to relay. */ @@ -100,6 +107,13 @@ class zmq_pub void operator()(std::uint64_t height, epee::span<const cryptonote::block> blocks) const; }; + //! Callable for `send_miner_data` with weak ownership to `zmq_pub` object. + struct miner_data + { + std::weak_ptr<zmq_pub> self_; + void operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const; + }; + //! Callable for `send_txpool_add` with weak ownership to `zmq_pub` object. struct txpool_add { diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 28e207ff2..b03da1edc 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -34,6 +34,7 @@ #include <type_traits> #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_core/cryptonote_tx_utils.h" // drop macro from windows.h #ifdef GetObject @@ -1411,6 +1412,27 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribu GET_FROM_JSON_OBJECT(val, dist.data.base, base); } +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry) +{ + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, id, entry.id); + INSERT_INTO_JSON_OBJECT(dest, weight, entry.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, entry.fee); + dest.EndObject(); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, entry.id, id); + GET_FROM_JSON_OBJECT(val, entry.weight, weight); + GET_FROM_JSON_OBJECT(val, entry.fee, fee); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 35ea990b3..c858faf5a 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry); +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 6e8d585dc..3b59267b2 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -49,6 +49,7 @@ using namespace epee; #include "cryptonote_core/tx_sanity_check.h" #include "wallet_rpc_helpers.h" #include "wallet2.h" +#include "wallet_args.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "net/parse.h" #include "rpc/core_rpc_server_commands_defs.h" @@ -276,7 +277,7 @@ struct options { const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; - const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; + const command_line::arg_descriptor<std::string> password_file = wallet_args::arg_password_file(); const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<std::string> daemon_login = {"daemon-login", tools::wallet2::tr("Specify username[:password] for daemon RPC client"), "", true}; const command_line::arg_descriptor<std::string> daemon_ssl = {"daemon-ssl", tools::wallet2::tr("Enable SSL on daemon RPC connections: enabled|disabled|autodetect"), "autodetect"}; @@ -532,7 +533,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl boost::optional<tools::password_container> get_password(const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char*, bool)> &password_prompter, const bool verify) { - if (command_line::has_arg(vm, opts.password) && command_line::has_arg(vm, opts.password_file)) + if (command_line::has_arg(vm, opts.password) && !command_line::is_arg_defaulted(vm, opts.password_file)) { THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, tools::wallet2::tr("can't specify more than one of --password and --password-file")); } @@ -542,10 +543,11 @@ boost::optional<tools::password_container> get_password(const boost::program_opt return tools::password_container{command_line::get_arg(vm, opts.password)}; } - if (command_line::has_arg(vm, opts.password_file)) + if (!command_line::is_arg_defaulted(vm, opts.password_file)) { std::string password; - bool r = epee::file_io_utils::load_file_to_string(command_line::get_arg(vm, opts.password_file), + const auto password_file = command_line::get_arg(vm, opts.password_file); + bool r = epee::file_io_utils::load_file_to_string(password_file, password); THROW_WALLET_EXCEPTION_IF(!r, tools::error::wallet_internal_error, tools::wallet2::tr("the password file specified could not be read")); @@ -1938,7 +1940,7 @@ void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::has const bool is_miner = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); if (!is_miner || m_refresh_type != RefreshType::RefreshNoCoinbase) { - const size_t rec_size = is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : tx.vout.size(); + const size_t rec_size = (is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase && tx.version < 2) ? 1 : tx.vout.size(); if (!tx.vout.empty()) { // if tx.vout is not empty, we loop through all tx pubkeys @@ -2087,7 +2089,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { // assume coinbase isn't for us } - else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) + else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase && tx.version < 2) { check_acc_out_precomp_once(tx.vout[0], derivation, additional_derivations, 0, is_out_data_ptr, tx_scan_info[0], output_found[0]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[0].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); @@ -2858,8 +2860,9 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry if (m_refresh_type != RefreshType::RefreshNoCoinbase) { THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); - const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); - tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + const cryptonote::transaction& tx = parsed_blocks[i].block.miner_tx; + const size_t n_vouts = (m_refresh_type == RefreshType::RefreshOptimizeCoinbase && tx.version < 2) ? 1 : tx.vout.size(); + tpool.submit(&waiter, [&, n_vouts, txidx](){ geniod(tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) @@ -4464,7 +4467,26 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_account.set_device(hwdev); account_public_address device_account_public_address; - THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + bool fetch_device_address = true; + + ::hw::device_cold* dev_cold = nullptr; + if (m_key_device_type == hw::device::device_type::TREZOR && (dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev)) != nullptr) { + THROW_WALLET_EXCEPTION_IF(!dev_cold->get_public_address_with_no_passphrase(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + if (device_account_public_address == m_account.get_keys().m_account_address) { + LOG_PRINT_L0("Wallet opened with an empty passphrase"); + fetch_device_address = false; + dev_cold->set_use_empty_passphrase(true); + } else { + fetch_device_address = true; + LOG_PRINT_L0("Wallet opening with an empty passphrase failed. Retry again: " << fetch_device_address); + dev_cold->reset_session(); + } + } + + if (fetch_device_address) { + THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + } + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. If the device uses the passphrase feature, please check whether the passphrase was entered correctly (it may have been misspelled - different passphrases generate different wallets, passphrase is case-sensitive). " "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + ", wallet address: " + m_account.get_public_address_str(m_nettype)); @@ -10260,6 +10282,38 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp const size_t num_outputs = get_num_outputs(tx.dsts, m_transfers, tx.selected_transfers); needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, num_outputs, extra.size(), bulletproof, clsag, base_fee, fee_multiplier, fee_quantization_mask); + auto try_carving_from_partial_payment = [&](uint64_t needed_fee, uint64_t available_for_fee) + { + // The check against original_output_index is to ensure the last entry in tx.dsts is really + // a partial payment. Otherwise multiple requested outputs to the same address could + // fool this logic into thinking there is a partial payment. + if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0 && tx.dsts.size() > original_output_index) + { + // we don't have enough for the fee, but we've only partially paid the current address, + // so we can take the fee from the paid amount, since we'll have to make another tx anyway + LOG_PRINT_L2("Attempting to carve tx fee " << print_money(needed_fee) << " from partial payment (first pass)"); + std::vector<cryptonote::tx_destination_entry>::iterator i; + i = std::find_if(tx.dsts.begin(), tx.dsts.end(), + [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &dsts[0].addr, sizeof(dsts[0].addr)); }); + THROW_WALLET_EXCEPTION_IF(i == tx.dsts.end(), error::wallet_internal_error, "paid address not found in outputs"); + if (i->amount > needed_fee) + { + uint64_t new_paid_amount = i->amount /*+ test_ptx.fee*/ - needed_fee; + LOG_PRINT_L2("Adjusting amount paid to " << get_account_address_as_str(m_nettype, i->is_subaddress, i->addr) << " from " << + print_money(i->amount) << " to " << print_money(new_paid_amount) << " to accommodate " << + print_money(needed_fee) << " fee"); + dsts[0].amount += i->amount - new_paid_amount; + i->amount = new_paid_amount; + test_ptx.fee = needed_fee; + available_for_fee = needed_fee; + } + } + return available_for_fee; + }; + + // Try to carve the estimated fee from the partial payment (if there is one) + available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); + uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); for (const auto &o: tx.dsts) outputs += o.amount; @@ -10285,26 +10339,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); - if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) - { - // we don't have enough for the fee, but we've only partially paid the current address, - // so we can take the fee from the paid amount, since we'll have to make another tx anyway - std::vector<cryptonote::tx_destination_entry>::iterator i; - i = std::find_if(tx.dsts.begin(), tx.dsts.end(), - [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &dsts[0].addr, sizeof(dsts[0].addr)); }); - THROW_WALLET_EXCEPTION_IF(i == tx.dsts.end(), error::wallet_internal_error, "paid address not found in outputs"); - if (i->amount > needed_fee) - { - uint64_t new_paid_amount = i->amount /*+ test_ptx.fee*/ - needed_fee; - LOG_PRINT_L2("Adjusting amount paid to " << get_account_address_as_str(m_nettype, i->is_subaddress, i->addr) << " from " << - print_money(i->amount) << " to " << print_money(new_paid_amount) << " to accommodate " << - print_money(needed_fee) << " fee"); - dsts[0].amount += i->amount - new_paid_amount; - i->amount = new_paid_amount; - test_ptx.fee = needed_fee; - available_for_fee = needed_fee; - } - } + // Try to carve the fee from the partial payment again after updating from estimate to actual + available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); if (needed_fee > available_for_fee) { diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 55058bf4e..066e98e52 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -80,6 +80,10 @@ namespace wallet_args { return {"rpc-client-secret-key", wallet_args::tr("Set RPC client secret key for RPC payments"), ""}; } + command_line::arg_descriptor<std::string> arg_password_file() + { + return {"password-file", wallet_args::tr("Wallet password file"), ""}; + } const char* tr(const char* str) { diff --git a/src/wallet/wallet_args.h b/src/wallet/wallet_args.h index 4af1b58fe..21e5f187c 100644 --- a/src/wallet/wallet_args.h +++ b/src/wallet/wallet_args.h @@ -37,6 +37,7 @@ namespace wallet_args command_line::arg_descriptor<std::string> arg_generate_from_json(); command_line::arg_descriptor<std::string> arg_wallet_file(); command_line::arg_descriptor<std::string> arg_rpc_client_secret_key(); + command_line::arg_descriptor<std::string> arg_password_file(); const char* tr(const char* str); diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index e1a06886b..4655e24cd 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -41,6 +41,7 @@ using namespace epee; #include "wallet/wallet_args.h" #include "common/command_line.h" #include "common/i18n.h" +#include "common/scoped_message_writer.h" #include "cryptonote_config.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" @@ -4525,10 +4526,12 @@ public: const auto arg_wallet_file = wallet_args::arg_wallet_file(); const auto arg_from_json = wallet_args::arg_generate_from_json(); const auto arg_rpc_client_secret_key = wallet_args::arg_rpc_client_secret_key(); + const auto arg_password_file = wallet_args::arg_password_file(); const auto wallet_file = command_line::get_arg(vm, arg_wallet_file); const auto from_json = command_line::get_arg(vm, arg_from_json); const auto wallet_dir = command_line::get_arg(vm, arg_wallet_dir); + const auto password_file = command_line::get_arg(vm, arg_password_file); const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password); const auto password_prompt = prompt_for_password ? password_prompter : nullptr; @@ -4538,6 +4541,12 @@ public: return false; } + if(!wallet_dir.empty() && !password_file.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("--password-file is not allowed in combination with --wallet-dir")); + return false; + } + if (!wallet_dir.empty()) { wal = NULL; @@ -4716,7 +4725,7 @@ int main(int argc, char** argv) { tools::wallet_rpc_server::tr("This is the RPC monero wallet. It needs to connect to a monero\ndaemon to work correctly."), desc_params, po::positional_options_description(), - [](const std::string &s, bool emphasis){ epee::set_console_color(emphasis ? epee::console_color_white : epee::console_color_default, true); std::cout << s << std::endl; if (emphasis) epee::reset_console_color(); }, + [](const std::string &s, bool emphasis){ tools::scoped_message_writer(emphasis ? epee::console_color_white : epee::console_color_default, true) << s; }, "monero-wallet-rpc.log", true ); diff --git a/tests/core_tests/transaction_tests.cpp b/tests/core_tests/transaction_tests.cpp index d19bdd1f9..efd294336 100644 --- a/tests/core_tests/transaction_tests.cpp +++ b/tests/core_tests/transaction_tests.cpp @@ -72,7 +72,6 @@ bool test_transaction_generation_and_ring_signature() construct_miner_tx(0, 0, 0, 0, 0, miner_acc6.get_keys().m_account_address, tx_mine_6); //fill inputs entry - typedef tx_source_entry::output_entry tx_output_entry; std::vector<tx_source_entry> sources; sources.resize(sources.size()+1); tx_source_entry& src = sources.back(); diff --git a/tests/unit_tests/logging.cpp b/tests/unit_tests/logging.cpp index f11b17412..b3ffb9aa6 100644 --- a/tests/unit_tests/logging.cpp +++ b/tests/unit_tests/logging.cpp @@ -208,3 +208,10 @@ TEST(logging, operator_equals_segfault) el::Logger log2("id2", nullptr); log2 = log1; } + +TEST(logging, empty_configurations_throws) +{ + el::Logger log1("id1", nullptr); + const el::Configurations cfg; + EXPECT_ANY_THROW(log1.configure(cfg)); +} diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index aa4f3ac0c..7907e9a9a 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -304,6 +304,9 @@ TEST(node_server, bind_same_p2p_port) Relevant part about REUSEADDR from man: https://www.man7.org/linux/man-pages/man7/ip.7.html + + For Mac OSX, set the following alias, before running the test, or else it will fail: + sudo ifconfig lo0 alias 127.0.0.2 */ vm.find(nodetool::arg_p2p_bind_ip.name)->second = boost::program_options::variable_value(std::string("127.0.0.2"), false); vm.find(nodetool::arg_p2p_bind_port.name)->second = boost::program_options::variable_value(std::string(port), false); diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 535752665..f4c73d3d5 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -132,7 +132,8 @@ TEST(Serialization, BinaryArchiveInts) { ASSERT_EQ(8, oss.str().size()); ASSERT_EQ(string("\0\0\0\0\xff\0\0\0", 8), oss.str()); - binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; + const std::string s = oss.str(); + binary_archive<false> iar{epee::strspan<std::uint8_t>(s)}; iar.serialize_int(x1); ASSERT_EQ(8, iar.getpos()); ASSERT_TRUE(iar.good()); @@ -150,7 +151,8 @@ TEST(Serialization, BinaryArchiveVarInts) { ASSERT_EQ(6, oss.str().size()); ASSERT_EQ(string("\x80\x80\x80\x80\xF0\x1F", 6), oss.str()); - binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; + const std::string s = oss.str(); + binary_archive<false> iar{epee::strspan<std::uint8_t>(s)}; iar.serialize_varint(x1); ASSERT_TRUE(iar.good()); ASSERT_EQ(x, x1); diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 207565e6f..435459f6d 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -53,6 +53,14 @@ class Daemon(object): return self.rpc.send_json_rpc_request(getblocktemplate) get_block_template = getblocktemplate + def get_miner_data(self): + get_miner_data = { + 'method': 'get_miner_data', + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_miner_data) + def add_aux_pow(self, blocktemplate_blob, aux_pow, client = ""): add_aux_pow = { 'method': 'add_aux_pow', |