diff options
40 files changed, 708 insertions, 278 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 806c8a0fc..b21c53401 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: continuous-integration/gh-actions/cli +name: ci/gh-actions/cli on: [push, pull_request] diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml new file mode 100644 index 000000000..85a05f673 --- /dev/null +++ b/.github/workflows/depends.yml @@ -0,0 +1,85 @@ +name: ci/gh-actions/depends + +on: [push, pull_request] + +jobs: + build-macos: + runs-on: ubuntu-18.04 + env: + CCACHE_COMPRESS: 1 + CCACHE_TEMPDIR: /tmp/.ccache-temp + strategy: + fail-fast: false + matrix: + toolchain: + - name: "RISCV 64bit" + host: "riscv64-linux-gnu" + packages: "python3 gperf g++-riscv64-linux-gnu" + - name: "ARM v7" + host: "arm-linux-gnueabihf" + packages: "python3 gperf g++-arm-linux-gnueabihf" + - name: "ARM v8" + host: "aarch64-linux-gnu" + packages: "python3 gperf g++-aarch64-linux-gnu" + - name: "i686 Win" + host: "i686-w64-mingw32" + packages: "python3 g++-mingw-w64-i686 qttools5-dev-tools" + - name: "i686 Linux" + host: "i686-pc-linux-gnu" + packages: "gperf cmake g++-multilib python3-zmq" + - name: "Win64" + host: "x86_64-w64-mingw32" + packages: "cmake python3 g++-mingw-w64-x86-64 qttools5-dev-tools" + - name: "x86_64 Linux" + host: "x86_64-unknown-linux-gnu" + packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" + - name: "Cross-Mac" + host: "x86_64-apple-darwin11" + packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" + osx_sdk: "10.11" + - name: "x86_64 Freebsd" + host: "x86_64-unknown-freebsd" + packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" + name: ${{ matrix.toolchain.name }} + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: ccache + uses: actions/cache@v2 + with: + path: | + ~/.ccache + contrib/depends/built + contrib/depends/sdk-sources + key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }} + restore-keys: ccache-${{ matrix.toolchain.host }}- + - name: set apt conf + run: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + - name: install dependencies + run: sudo apt -y install build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache ${{ matrix.toolchain.packages }} + - name: prepare apple-darwin11 + if: ${{ matrix.toolchain.host == 'x86_64-apple-darwin11' }} + run: | + mkdir -p contrib/depends/SDKs contrib/depends/sdk-sources + if [ ! -f contrib/depends/sdk-sources/MacOSX${{ matrix.toolchain.osx_sdk }}.sdk.tar.gz ]; then curl --location --fail https://bitcoincore.org/depends-sources/sdks/MacOSX${{ matrix.toolchain.osx_sdk }}.sdk.tar.gz -o contrib/depends/sdk-sources/MacOSX${{ matrix.toolchain.osx_sdk }}.sdk.tar.gz; fi + if [ -f contrib/depends/sdk-sources/MacOSX${{ matrix.toolchain.osx_sdk }}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${{ matrix.toolchain.osx_sdk }}.sdk.tar.gz; fi + - name: prepare w64-mingw32 + if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'i686-w64-mingw32' }} + run: | + sudo update-alternatives --set ${{ matrix.toolchain.host }}-g++ $(which ${{ matrix.toolchain.host }}-g++-posix) + sudo update-alternatives --set ${{ matrix.toolchain.host }}-gcc $(which ${{ matrix.toolchain.host }}-gcc-posix) + - name: build + run: | + ccache --max-size=150M + make depends target=${{ matrix.toolchain.host }} -j2 + - uses: actions/upload-artifact@v2 + if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} + with: + name: ${{ matrix.toolchain.name }} + path: | + /home/runner/work/monero/monero/build/${{ matrix.toolchain.host }}/release/bin/monero-wallet-cli* + /home/runner/work/monero/monero/build/${{ matrix.toolchain.host }}/release/bin/monerod* diff --git a/.gitignore b/.gitignore index 08c310e66..049fc2562 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ cscope.po.out external/miniupnpc/Makefile miniupnpcstrings.h version/ +ClangBuildAnalyzerSession.txt # Created by https://www.gitignore.io ### C++ ### diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4d49e3539..000000000 --- a/.travis.yml +++ /dev/null @@ -1,66 +0,0 @@ -sudo: required -dist: trusty -os: linux -language: minimal -cache: - directories: - - contrib/depends/built - - contrib/depends/sdk-sources - - $HOME/.ccache -env: - global: - - MAKEJOBS=-j3 - - CCACHE_SIZE=100M - - CCACHE_TEMPDIR=/tmp/.ccache-temp - - CCACHE_COMPRESS=1 - - CCACHE_DIR=$HOME/.ccache - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - DOCKER_PACKAGES="build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" - matrix: -# RISCV 64bit - - HOST=riscv64-linux-gnu PACKAGES="python3 gperf g++-riscv64-linux-gnu" -# ARM v7 - - HOST=arm-linux-gnueabihf PACKAGES="python3 gperf g++-arm-linux-gnueabihf" -# ARM v8 - - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu" -# i686 Win - - HOST=i686-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="python3 g++-mingw-w64-i686 qttools5-dev-tools" MAKEJOBS=-j2 -# i686 Linux - - HOST=i686-pc-linux-gnu PACKAGES="gperf cmake g++-multilib python3-zmq" -# Win64 - - HOST=x86_64-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="cmake python3 g++-mingw-w64-x86-64 qttools5-dev-tools" -# x86_64 Linux - - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" -# Cross-Mac - - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" OSX_SDK=10.11 -# x86_64 Freebsd - - HOST=x86_64-unknown-freebsd PACKAGES="clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" - -before_install: - - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") -install: - - env | grep -E '^(CCACHE_|DISPLAY|CONFIG_SHELL)' | tee /tmp/env - - if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN"; fi - - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env ubuntu:18.04) - - DOCKER_EXEC="docker exec $DOCKER_ID" - - if [ -n "$DPKG_ADD_ARCH" ]; then $DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - - travis_retry $DOCKER_EXEC apt-get update - - travis_retry $DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES -before_script: - - mkdir -p contrib/depends/SDKs contrib/depends/sdk-sources - - if [ -n "$OSX_SDK" -a ! -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - - if [ -n "$OSX_SDK" -a -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-g++ \$(which $HOST-g++-posix)"; fi - - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-gcc \$(which $HOST-gcc-posix)"; fi - - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi -script: - - git submodule init && git submodule update - - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` - - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC ccache --max-size=$CCACHE_SIZE; fi - - $DOCKER_EXEC bash -c "mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/share/toolchain.cmake .. && make $MAKEJOBS" - - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/lib -after_script: - - echo $TRAVIS_COMMIT_RANGE - - echo $TRAVIS_COMMIT_LOG diff --git a/CMakeLists.txt b/CMakeLists.txt index 050269218..8a7b783aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,13 @@ if (USE_CCACHE) else() message(STATUS "ccache deselected") endif() +option (USE_COMPILATION_TIME_PROFILER "Use compilation time profiler (for CLang >= 9 only)" OFF) +if (USE_COMPILATION_TIME_PROFILER) + if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + message(FATAL_ERROR "The flag USE_COMPILATION_TIME_PROFILER is meant to be set only for CLang compiler!") + endif() + add_compile_options("-ftime-trace") +endif() if (${CMAKE_VERSION} VERSION_GREATER "3.0.0" AND CMAKE_MAKE_PROGRAM MATCHES "ninja") set(MONERO_PARALLEL_COMPILE_JOBS "" CACHE STRING "The maximum number of concurrent compilation jobs.") @@ -67,6 +74,20 @@ if (${CMAKE_VERSION} VERSION_GREATER "3.0.0" AND CMAKE_MAKE_PROGRAM MATCHES "nin set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${MONERO_PARALLEL_LINK_JOBS}) set(CMAKE_JOB_POOL_LINK link_job_pool) endif () +endif () + +option (USE_CLANG_TIDY_C "Lint the code with clang-tidy - variant C" OFF) +option (USE_CLANG_TIDY_CXX "Lint the code with clang-tidy - variant C++" OFF) +if (USE_CLANG_TIDY_C AND USE_CLANG_TIDY_CXX) + message(FATAL_ERROR "Enabling both USE_CLANG_TIDY_C and USE_CLANG_TIDY_CXX simultaneously crashes clang-tidy.") +endif() +if (USE_CLANG_TIDY_C OR USE_CLANG_TIDY_CXX) + include(SetClangTidy) +endif() +if (USE_CLANG_TIDY_C) + monero_clang_tidy("C") +elseif (USE_CLANG_TIDY_CXX) + monero_clang_tidy("CXX") endif() enable_language(C ASM) @@ -687,6 +687,9 @@ Example command line to start monerod through Tor: DNS_PUBLIC=tcp torsocks monerod --p2p-bind-ip 127.0.0.1 --no-igd ``` +A helper script is in contrib/tor/monero-over-tor.sh. It assumes Tor is installed +already, and runs Tor and Monero with the right configuration. + ### Using Tor on Tails TAILS ships with a very restrictive set of firewall rules. Therefore, you need diff --git a/cmake/SetClangTidy.cmake b/cmake/SetClangTidy.cmake new file mode 100644 index 000000000..ce66286e2 --- /dev/null +++ b/cmake/SetClangTidy.cmake @@ -0,0 +1,72 @@ +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_CLANG_TIDY.html +# This module sets the following variables: +# CMAKE_C_CLANG_TIDY +# CMAKE_CXX_CLANG_TIDY +# when clang-tidy is found in PATH. Afterwards, the code is being linted by the tool. +# The checks to be enabled can be manipulated with the variable MONERO_CLANG_TIDY_CHECKS + +macro (monero_clang_tidy LANGUAGE) + set(TOOL_NAME "clang-tidy") + set(MONERO_CLANG_TIDY_MIN_VERSION "3.6") + if(${CMAKE_VERSION} VERSION_LESS "${MONERO_CLANG_TIDY_MIN_VERSION}") + message(FATAL_ERROR "Sorry, ${TOOL_NAME} is available for CMake from version ${MONERO_CLANG_TIDY_MIN_VERSION}") + else() + message(STATUS "Trying to enable ${TOOL_NAME}") + find_program(MONERO_CLANG_BIN ${TOOL_NAME}) + if(NOT MONERO_CLANG_BIN) + message(FATAL_ERROR "${TOOL_NAME} not found! Try running: sudo apt install ${TOOL_NAME}") + else() + message(STATUS "Found ${MONERO_CLANG_BIN}") + set(MONERO_CLANG_TIDY_CHECKS + -header-filter=.; # By default the headers are excluded. This line enables them. + -checks=*; # Currently enabling all checks + # An example of selectively enabling checks: + #-checks=bugprone-*,cppcoreguidelines-avoid-goto # Have to be in one line :( + ) + # Current list of checks is avaibale under: + # https://clang.llvm.org/extra/clang-tidy/ + if (${LANGUAGE} STREQUAL "C") + set(CMAKE_C_CLANG_TIDY + ${MONERO_CLANG_BIN}; # Mind the semicolon + ${MONERO_CLANG_TIDY_CHECKS} + ) + elseif (${LANGUAGE} STREQUAL "CXX") + set(CMAKE_CXX_CLANG_TIDY + ${MONERO_CLANG_BIN}; # Mind the semicolon + ${MONERO_CLANG_TIDY_CHECKS} + ) + else() + message(FATAL_ERROR "${TOOL_NAME}: Unsupported language: ${LANGUAGE}") + endif() + endif() + endif() +endmacro() + diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 9d3c28465..62e975e50 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,36 +1,31 @@ package=openssl -$(package)_version=1.0.2r -$(package)_download_path=https://ftp.openssl.org/source/old/1.0.2 +$(package)_version=1.1.1i +$(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6 -$(package)_patches=fix_arflags.patch +$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" +$(package)_config_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib +$(package)_config_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib +$(package)_build_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native" +$(package)_build_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native" $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl $(package)_config_opts+=no-capieng $(package)_config_opts+=no-dso $(package)_config_opts+=no-dtls1 $(package)_config_opts+=no-ec_nistp_64_gcc_128 $(package)_config_opts+=no-gost -$(package)_config_opts+=no-gmp $(package)_config_opts+=no-heartbeats -$(package)_config_opts+=no-jpake -$(package)_config_opts+=no-krb5 -$(package)_config_opts+=no-libunbound $(package)_config_opts+=no-md2 $(package)_config_opts+=no-rc5 $(package)_config_opts+=no-rdrand $(package)_config_opts+=no-rfc3779 -$(package)_config_opts+=no-rsax $(package)_config_opts+=no-sctp -$(package)_config_opts+=no-sha0 $(package)_config_opts+=no-shared $(package)_config_opts+=no-ssl-trace $(package)_config_opts+=no-ssl2 $(package)_config_opts+=no-ssl3 -$(package)_config_opts+=no-static_engine -$(package)_config_opts+=no-store $(package)_config_opts+=no-unit-test $(package)_config_opts+=no-weak-ssl-ciphers $(package)_config_opts+=no-zlib @@ -42,8 +37,8 @@ $(package)_config_opts_x86_64_linux=linux-x86_64 $(package)_config_opts_i686_linux=linux-generic32 $(package)_config_opts_arm_linux=linux-generic32 $(package)_config_opts_aarch64_linux=linux-generic64 -$(package)_config_opts_arm_android=--static android-armv7 no-asm -$(package)_config_opts_aarch64_android=--static android no-asm +$(package)_config_opts_arm_android=--static android-arm +$(package)_config_opts_aarch64_android=--static android-arm64 $(package)_config_opts_riscv64_linux=linux-generic64 $(package)_config_opts_mipsel_linux=linux-generic32 $(package)_config_opts_mips_linux=linux-generic32 @@ -55,10 +50,8 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64 endef define $(package)_preprocess_cmds - sed -i.old "/define DATE/d" util/mkbuildinf.pl && \ - sed -i.old "s|engines apps test|engines|" Makefile.org && \ - sed -i -e "s/-mandroid //" Configure && \ - patch < $($(package)_patch_dir)/fix_arflags.patch + sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \ + sed -i -e 's|cflags --sysroot.*",|cflags",|' Configurations/15-android.conf endef define $(package)_config_cmds @@ -70,7 +63,7 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw + $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw endef define $(package)_postprocess_cmds diff --git a/contrib/depends/patches/openssl/fix_arflags.patch b/contrib/depends/patches/openssl/fix_arflags.patch deleted file mode 100644 index 2d2900d80..000000000 --- a/contrib/depends/patches/openssl/fix_arflags.patch +++ /dev/null @@ -1,24 +0,0 @@ ---- Makefile.org.O 2019-02-26 14:20:20.000000000 +0000 -+++ Makefile.org 2019-11-15 13:05:54.370086856 +0000 -@@ -63,8 +63,8 @@ - PEX_LIBS= - EX_LIBS= - EXE_EXT= --ARFLAGS= --AR=ar $(ARFLAGS) r -+ARFLAGS= r -+AR=ar $(ARFLAGS) - RANLIB= ranlib - RC= windres - NM= nm ---- Configure.O 2019-02-26 14:20:20.000000000 +0000 -+++ Configure 2019-11-16 07:43:14.933990774 +0000 -@@ -1251,7 +1251,7 @@ - my $shared_extension = $fields[$idx_shared_extension]; - my $ranlib = $ENV{'RANLIB'} || $fields[$idx_ranlib]; - my $ar = $ENV{'AR'} || "ar"; --my $arflags = $fields[$idx_arflags]; -+my $arflags = $ENV{'ARFLAGS'} || $fields[$idx_arflags]; - my $windres = $ENV{'RC'} || $ENV{'WINDRES'} || "windres"; - my $multilib = $fields[$idx_multilib]; - diff --git a/contrib/epee/include/byte_stream.h b/contrib/epee/include/byte_stream.h index 42a9e1dd9..30abd050d 100644 --- a/contrib/epee/include/byte_stream.h +++ b/contrib/epee/include/byte_stream.h @@ -58,7 +58,6 @@ namespace epee byte_buffer buffer_; //! Beginning of buffer std::uint8_t* next_write_; //! Current write position const std::uint8_t* end_; //! End of buffer - std::size_t increase_size_; //! Minimum buffer size increase //! \post `requested <= available()` void overflow(const std::size_t requested); @@ -75,29 +74,17 @@ namespace epee using char_type = std::uint8_t; using Ch = char_type; - //! \return Default minimum size increase on buffer overflow - static constexpr std::size_t default_increase() noexcept { return 4096; } - //! Increase internal buffer by at least `byte_stream_increase` bytes. byte_stream() noexcept - : byte_stream(default_increase()) - {} - - //! Increase internal buffer by at least `increase` bytes. - explicit byte_stream(const std::size_t increase) noexcept : buffer_(nullptr), next_write_(nullptr), - end_(nullptr), - increase_size_(increase) + end_(nullptr) {} byte_stream(byte_stream&& rhs) noexcept; ~byte_stream() noexcept = default; byte_stream& operator=(byte_stream&& rhs) noexcept; - //! \return The minimum increase size on buffer overflow - std::size_t increase_size() const noexcept { return increase_size_; } - const std::uint8_t* data() const noexcept { return buffer_.get(); } std::uint8_t* tellp() const noexcept { return next_write_; } std::size_t available() const noexcept { return end_ - next_write_; } diff --git a/contrib/epee/include/net/abstract_http_client.h b/contrib/epee/include/net/abstract_http_client.h index 1f8bbc605..5270824e1 100644 --- a/contrib/epee/include/net/abstract_http_client.h +++ b/contrib/epee/include/net/abstract_http_client.h @@ -70,7 +70,7 @@ namespace http virtual bool connect(std::chrono::milliseconds timeout) = 0; virtual bool disconnect() = 0; virtual bool is_connected(bool *ssl = NULL) = 0; - virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; + virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual uint64_t get_bytes_sent() const = 0; diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 9645e896b..3725ef079 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -233,7 +233,7 @@ namespace net_utils } //--------------------------------------------------------------------------- - inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override + inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override { CRITICAL_REGION_LOCAL(m_lock); if(!is_connected()) diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index ac22cd7a9..1665fdac7 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -118,8 +118,10 @@ return true; \ } \ uint64_t ticks2 = misc_utils::get_tick_count(); \ - epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), response_info.m_body); \ + epee::byte_slice buffer; \ + epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), buffer, 64 * 1024); \ uint64_t ticks3 = epee::misc_utils::get_tick_count(); \ + response_info.m_body.assign(reinterpret_cast<const char*>(buffer.data()), buffer.size()); \ response_info.m_mime_tipe = " application/octet-stream"; \ response_info.m_header_info.m_content_type = " application/octet-stream"; \ MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \ diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index f9b6f9a81..ad561c5b6 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,7 +31,6 @@ #include <cstdint> -#include "byte_slice.h" #include "net_utils_base.h" #include "span.h" @@ -39,6 +38,7 @@ namespace epee { +class byte_slice; namespace levin { #pragma pack(push) @@ -86,7 +86,7 @@ namespace levin template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 1341a4ae6..8cb2be3e1 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -514,16 +514,15 @@ public: { if(m_current_head.m_have_to_return_data) { - std::string return_buff; + byte_slice return_buff; const uint32_t return_code = m_config.m_pcommands_handler->invoke( m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context ); bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); head.m_return_code = SWAP32LE(return_code); - return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head)); - if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)})) + if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) return false; MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 9446e3588..508c79d76 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -44,6 +44,7 @@ #include <boost/lambda/lambda.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/system/error_code.hpp> +#include <boost/utility/string_ref.hpp> #include <functional> #include "net/net_utils_base.h" #include "net/net_ssl.h" @@ -280,7 +281,7 @@ namespace net_utils inline - bool send(const std::string& buff, std::chrono::milliseconds timeout) + bool send(const boost::string_ref buff, std::chrono::milliseconds timeout) { try @@ -298,7 +299,7 @@ namespace net_utils // object is used as a callback and will update the ec variable when the // operation completes. The blocking_udp_client.cpp example shows how you // can use boost::bind rather than boost::lambda. - async_write(buff.c_str(), buff.size(), ec); + async_write(buff.data(), buff.size(), ec); // Block until the asynchronous operation has completed. while (ec == boost::asio::error::would_block) diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index a8bc945a8..c4cb91130 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -29,6 +29,7 @@ #include <boost/utility/string_ref.hpp> #include <chrono> #include <string> +#include "byte_slice.h" #include "portable_storage_template_helper.h" #include "net/http_base.h" #include "net/http_server_handlers_map2.h" @@ -74,12 +75,12 @@ namespace epee template<class t_request, class t_response, class t_transport> bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST") { - std::string req_param; - if(!serialization::store_t_to_binary(out_struct, req_param)) + byte_slice req_param; + if(!serialization::store_t_to_binary(out_struct, req_param, 16 * 1024)) return false; const http::http_response_info* pri = NULL; - if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri))) + if(!transport.invoke(uri, method, boost::string_ref{reinterpret_cast<const char*>(req_param.data()), req_param.size()}, timeout, std::addressof(pri))) { LOG_PRINT_L1("Failed to invoke http request to " << uri); return false; diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index cf1262486..95f0bb410 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -27,8 +27,10 @@ #pragma once #include "portable_storage_template_helper.h" +#include <boost/utility/string_ref.hpp> #include <boost/utility/value_init.hpp> #include <functional> +#include "byte_slice.h" #include "span.h" #include "net/levin_base.h" @@ -110,11 +112,12 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - std::string buff_to_send, buff_to_recv; - stg.store_to_binary(buff_to_send); + byte_slice buff_to_send; + std::string buff_to_recv; + stg.store_to_binary(buff_to_send, 16 * 1024); on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id); + int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); @@ -137,10 +140,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - std::string buff_to_send; - stg.store_to_binary(buff_to_send); + byte_slice buff_to_send; + stg.store_to_binary(buff_to_send, 16 * 1024); on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -184,11 +187,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - std::string buff_to_send; + byte_slice buff_to_send; stg.store_to_binary(buff_to_send); on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id); + int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -199,7 +202,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) @@ -222,7 +225,7 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out)) + if(!strg_out.store_to_binary(buff_out, 32 * 1024)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; @@ -254,7 +257,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -263,13 +266,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; std::string fake_str;\ + bool handled = false; epee::byte_slice fake_str; \ return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -289,7 +292,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index 4b759a24f..589e6ad63 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -32,7 +32,6 @@ #include "misc_language.h" #include "portable_storage_base.h" -#include "portable_storage_to_bin.h" #include "portable_storage_from_bin.h" #include "portable_storage_to_json.h" #include "portable_storage_from_json.h" @@ -42,6 +41,7 @@ namespace epee { + class byte_slice; namespace serialization { /************************************************************************/ @@ -83,7 +83,7 @@ namespace epee bool delete_entry(const std::string& pentry_name, hsection hparent_section = nullptr); //------------------------------------------------------------------------------- - bool store_to_binary(binarybuffer& target); + bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); bool load_from_binary(const epee::span<const uint8_t> target); bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); } template<class trace_policy> @@ -133,22 +133,6 @@ namespace epee { return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format } - - inline - bool portable_storage::store_to_binary(binarybuffer& target) - { - TRY_ENTRY(); - std::stringstream ss; - storage_block_header sbh = AUTO_VAL_INIT(sbh); - sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); - sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); - sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; - ss.write((const char*)&sbh, sizeof(storage_block_header)); - pack_entry_to_buff(ss, m_root); - target = ss.str(); - return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) - } inline bool portable_storage::load_from_binary(const epee::span<const uint8_t> source) { diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 13c870d44..39f900c8d 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -28,6 +28,7 @@ #include <string> +#include "byte_slice.h" #include "parserse_base_utils.h" #include "portable_storage.h" #include "file_io_utils.h" @@ -111,18 +112,18 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - bool store_t_to_binary(t_struct& str_in, std::string& binary_buff, size_t indent = 0) + bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff, size_t initial_buffer_size = 8192) { portable_storage ps; str_in.store(ps); - return ps.store_to_binary(binary_buff); + return ps.store_to_binary(binary_buff, initial_buffer_size); } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - std::string store_t_to_binary(t_struct& str_in, size_t indent = 0) + byte_slice store_t_to_binary(t_struct& str_in, size_t initial_buffer_size = 8192) { - std::string binary_buff; - store_t_to_binary(str_in, binary_buff, indent); + byte_slice binary_buff; + store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } } diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 8adf69162..5e101a86a 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -29,7 +29,7 @@ add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp - int-util.cpp) + int-util.cpp portable_storage.cpp) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) add_library(epee_readline STATIC readline_buffer.cpp) diff --git a/contrib/epee/src/byte_stream.cpp b/contrib/epee/src/byte_stream.cpp index e87d9f0bc..73bba92f2 100644 --- a/contrib/epee/src/byte_stream.cpp +++ b/contrib/epee/src/byte_stream.cpp @@ -34,6 +34,11 @@ #include <iostream> +namespace +{ + constexpr const std::size_t minimum_increase = 4096; +} + namespace epee { void byte_stream::overflow(const std::size_t requested) @@ -46,7 +51,7 @@ namespace epee const std::size_t len = size(); const std::size_t cap = capacity(); - const std::size_t increase = std::max(need, increase_size()); + const std::size_t increase = std::max(std::max(need, cap), minimum_increase); next_write_ = nullptr; end_ = nullptr; @@ -62,8 +67,7 @@ namespace epee byte_stream::byte_stream(byte_stream&& rhs) noexcept : buffer_(std::move(rhs.buffer_)), next_write_(rhs.next_write_), - end_(rhs.end_), - increase_size_(rhs.increase_size_) + end_(rhs.end_) { rhs.next_write_ = nullptr; rhs.end_ = nullptr; @@ -76,7 +80,6 @@ namespace epee buffer_ = std::move(rhs.buffer_); next_write_ = rhs.next_write_; end_ = rhs.end_; - increase_size_ = rhs.increase_size_; rhs.next_write_ = nullptr; rhs.end_ = nullptr; } diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp new file mode 100644 index 000000000..4534deff3 --- /dev/null +++ b/contrib/epee/src/portable_storage.cpp @@ -0,0 +1,29 @@ + +#include "byte_slice.h" +#include "byte_stream.h" +#include "misc_log_ex.h" +#include "span.h" +#include "storages/portable_storage.h" +#include "storages/portable_storage_to_bin.h" + +namespace epee +{ +namespace serialization +{ + bool portable_storage::store_to_binary(byte_slice& target, const std::size_t initial_buffer_size) + { + TRY_ENTRY(); + byte_stream ss; + ss.reserve(initial_buffer_size); + storage_block_header sbh{}; + sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); + sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); + sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; + ss.write(epee::as_byte_span(sbh)); + pack_entry_to_buff(ss, m_root); + target = epee::byte_slice{std::move(ss)}; + return true; + CATCH_ENTRY("portable_storage::store_to_binary", false) + } +} +} diff --git a/contrib/tor/monero-over-tor.sh b/contrib/tor/monero-over-tor.sh new file mode 100755 index 000000000..9fb4cdd36 --- /dev/null +++ b/contrib/tor/monero-over-tor.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +DIR=$(realpath $(dirname $0)) + +echo "Checking monerod..." +monerod="" +for dir in \ + . \ + "$DIR" \ + "$DIR/../.." \ + "$DIR/build/release/bin" \ + "$DIR/../../build/release/bin" \ + "$DIR/build/Linux/master/release/bin" \ + "$DIR/../../build/Linux/master/release/bin" \ + "$DIR/build/Windows/master/release/bin" \ + "$DIR/../../build/Windows/master/release/bin" +do + if test -x "$dir/monerod" + then + monerod="$dir/monerod" + break + fi +done +if test -z "$monerod" +then + echo "monerod not found" + exit 1 +fi +echo "Found: $monerod" + +TORDIR="$DIR/monero-over-tor" +TORRC="$TORDIR/torrc" +HOSTNAMEFILE="$TORDIR/hostname" +echo "Creating configuration..." +mkdir -p "$TORDIR" +chmod 700 "$TORDIR" +rm -f "$TORRC" +cat << EOF > "$TORRC" +ControlSocket $TORDIR/control +ControlSocketsGroupWritable 1 +CookieAuthentication 1 +CookieAuthFile $TORDIR/control.authcookie +CookieAuthFileGroupReadable 1 +HiddenServiceDir $TORDIR +HiddenServicePort 18083 127.0.0.1:18083 +EOF + +echo "Starting Tor..." +nohup tor -f "$TORRC" 2> "$TORDIR/tor.stderr" 1> "$TORDIR/tor.stdout" & +ready=0 +for i in `seq 10` +do + sleep 1 + if test -f "$HOSTNAMEFILE" + then + ready=1 + break + fi +done +if test "$ready" = 0 +then + echo "Error starting Tor" + cat "$TORDIR/tor.stdout" + exit 1 +fi + +echo "Starting monerod..." +HOSTNAME=$(cat "$HOSTNAMEFILE") +"$monerod" \ + --anonymous-inbound "$HOSTNAME":18083,127.0.0.1:18083,25 --tx-proxy tor,127.0.0.1:9050,10 \ + --add-priority-node zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083 \ + --add-priority-node 2xmrnode5itf65lz.onion:18083 \ + --detach +ready=0 +for i in `seq 10` +do + sleep 1 + status=$("$monerod" status) + echo "$status" | grep -q "Height:" + if test $? = 0 + then + ready=1 + break + fi +done +if test "$ready" = 0 +then + echo "Error starting monerod" + tail -n 400 "$HOME/.bitmonero/bitmonero.log" | grep -Ev stacktrace\|"Error: Couldn't connect to daemon:"\|"src/daemon/main.cpp:.*Monero\ \'" | tail -n 20 + exit 1 +fi + +echo "Ready. Your Tor hidden service is $HOSTNAME" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index b416b86c8..6368f8190 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -37,6 +37,7 @@ #include <boost/program_options/variables_map.hpp> #include <string> +#include "byte_slice.h" #include "math_helper.h" #include "storages/levin_abstract_invoke2.h" #include "warnings.h" @@ -100,7 +101,7 @@ namespace cryptonote void set_p2p_endpoint(nodetool::i_p2p_endpoint<connection_context>* p2p); //bool process_handshake_data(const blobdata& data, cryptonote_connection_context& context); bool process_payload_sync_data(const CORE_SYNC_DATA& hshd, cryptonote_connection_context& context, bool is_inital); - bool get_payload_sync_data(blobdata& data); + bool get_payload_sync_data(epee::byte_slice& data); bool get_payload_sync_data(CORE_SYNC_DATA& hshd); bool on_callback(cryptonote_connection_context& context); t_core& get_core(){return m_core;} @@ -192,10 +193,10 @@ namespace cryptonote bool post_notify(typename t_parameter::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parameter).name() << " -->"); - std::string blob; - epee::serialization::store_t_to_binary(arg, blob); + epee::byte_slice blob; + epee::serialization::store_t_to_binary(arg, blob, 256 * 1024); // optimize for block responses //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan<uint8_t>(blob), context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::to_span(blob), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index cb743316a..bf7b061d9 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -425,7 +425,7 @@ namespace cryptonote } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> - bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(blobdata& data) + bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(epee::byte_slice& data) { CORE_SYNC_DATA hsd = {}; get_payload_sync_data(hsd); @@ -2491,7 +2491,7 @@ skip: drop_connection(context, true, false); return 1; } - if (arg.total_height < arg.m_block_ids.size() || arg.start_height > arg.total_height - arg.m_block_ids.size()) + if (arg.total_height < arg.m_block_ids.size() || arg.start_height > arg.total_height - arg.m_block_ids.size() || arg.start_height >= m_core.get_current_blockchain_height()) { LOG_ERROR_CCONTEXT("sent invalid start/nblocks/height, dropping connection"); drop_connection(context, true, false); @@ -2537,8 +2537,15 @@ skip: context.m_needed_objects.clear(); uint64_t added = 0; + std::unordered_set<crypto::hash> blocks_found; for (size_t i = 0; i < arg.m_block_ids.size(); ++i) { + if (!blocks_found.insert(arg.m_block_ids[i]).second) + { + LOG_ERROR_CCONTEXT("Duplicate blocks in chain entry response, dropping connection"); + drop_connection(context, true, false); + return 1; + } const uint64_t block_weight = arg.m_block_weights.empty() ? 0 : arg.m_block_weights[i]; context.m_needed_objects.push_back(std::make_pair(arg.m_block_ids[i], block_weight)); if (++added == n_use_blocks) @@ -2591,15 +2598,15 @@ skip: // send fluffy ones first, we want to encourage people to run that if (!fluffyConnections.empty()) { - std::string fluffyBlob; - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan<uint8_t>(fluffyBlob), std::move(fluffyConnections)); + epee::byte_slice fluffyBlob; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob, 32 * 1024); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::to_span(fluffyBlob), std::move(fluffyConnections)); } if (!fullConnections.empty()) { - std::string fullBlob; - epee::serialization::store_t_to_binary(arg, fullBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan<uint8_t>(fullBlob), std::move(fullConnections)); + epee::byte_slice fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob, 128 * 1024); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::to_span(fullBlob), std::move(fullConnections)); } return true; diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 2d04dffb6..318bcf4e1 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -36,6 +36,7 @@ #include <stdexcept> #include <utility> +#include "byte_slice.h" #include "common/expect.h" #include "common/varint.h" #include "cryptonote_config.h" @@ -52,7 +53,7 @@ namespace { - int get_command_from_message(const cryptonote::blobdata &msg) + int get_command_from_message(const epee::byte_slice &msg) { return msg.size() >= sizeof(epee::levin::bucket_head2) ? SWAP32LE(((epee::levin::bucket_head2*)msg.data())->m_command) : 0; } @@ -166,7 +167,7 @@ namespace levin return get_out_connections(p2p, get_blockchain_height(p2p, core)); } - std::string make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) + epee::byte_slice make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) { NOTIFY_NEW_TRANSACTIONS::request request{}; request.txs = std::move(txs); @@ -188,7 +189,7 @@ namespace levin padding -= overhead; request._ = std::string(padding, ' '); - std::string arg_buff; + epee::byte_slice arg_buff; epee::serialization::store_t_to_binary(request, arg_buff); // we probably lowballed the payload size a bit, so added a but too much. Fix this now. @@ -200,7 +201,7 @@ namespace levin // if the size of _ moved enough, we might lose byte in size encoding, we don't care } - std::string fullBlob; + epee::byte_slice fullBlob; if (!epee::serialization::store_t_to_binary(request, fullBlob)) throw std::runtime_error{"Failed to serialize to epee binary format"}; @@ -209,12 +210,12 @@ namespace levin bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff) { - const cryptonote::blobdata blob = make_tx_payload(std::move(txs), pad, fluff); + const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); p2p.for_connection(destination, [&blob](detail::p2p_context& context) { on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob)); return true; }); - return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(blob), destination); + return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); } /* The current design uses `asio::strand`s. The documentation isn't as clear @@ -825,9 +826,9 @@ namespace levin // Padding is not useful when using noise mode. Send as stem so receiver // forwards in Dandelion++ mode. - const std::string payload = make_tx_payload(std::move(txs), false, false); + const epee::byte_slice payload = make_tx_payload(std::move(txs), false, false); epee::byte_slice message = epee::levin::make_fragmented_notify( - zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(payload) + zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(payload) ); if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size()) { diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 920e0413c..91b3c0535 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -3788,7 +3788,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable //---------------------------------------------------------------------------------------------------- boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only) { - std::string account_data; + epee::byte_slice account_data; std::string multisig_signers; std::string multisig_derivations; cryptonote::account_base account = m_account; @@ -3815,7 +3815,7 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee: rapidjson::Document json; json.SetObject(); rapidjson::Value value(rapidjson::kStringType); - value.SetString(account_data.c_str(), account_data.length()); + value.SetString(reinterpret_cast<const char*>(account_data.data()), account_data.size()); json.AddMember("key_data", value, json.GetAllocator()); if (!seed_language.empty()) { @@ -3989,13 +3989,12 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee: rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); json.Accept(writer); - account_data = buffer.GetString(); // Encrypt the entire JSON object. std::string cipher; - cipher.resize(account_data.size()); + cipher.resize(buffer.GetSize()); keys_file_data.get().iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.get().iv, &cipher[0]); + crypto::chacha20(buffer.GetString(), buffer.GetSize(), key, keys_file_data.get().iv, &cipher[0]); keys_file_data.get().account_data = cipher; return keys_file_data; } diff --git a/tests/fuzz/http-client.cpp b/tests/fuzz/http-client.cpp index 4b12b36bb..65df24a3f 100644 --- a/tests/fuzz/http-client.cpp +++ b/tests/fuzz/http-client.cpp @@ -26,6 +26,7 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include <boost/utility/string_ref.hpp> #include "include_base_utils.h" #include "file_io_utils.h" #include "net/http_client.h" @@ -38,7 +39,7 @@ public: bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout, bool ssl = false, const std::string& bind_ip = "0.0.0.0") { return true; } bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, bool ssl = false, const std::string& bind_ip = "0.0.0.0") { return true; } bool disconnect() { return true; } - bool send(const std::string& buff, std::chrono::milliseconds timeout) { return true; } + bool send(const boost::string_ref buff, std::chrono::milliseconds timeout) { return true; } bool send(const void* data, size_t sz) { return true; } bool is_connected() { return true; } bool recv(std::string& buff, std::chrono::milliseconds timeout) diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index 012d05f36..b090c350b 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -65,13 +65,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf; + buff_out = m_invoke_out_buf.clone(); return m_return_code; } @@ -111,8 +111,7 @@ namespace int return_code() const { return m_return_code; } void return_code(int v) { m_return_code = v; } - const std::string& invoke_out_buf() const { return m_invoke_out_buf; } - void invoke_out_buf(const std::string& v) { m_invoke_out_buf = v; } + void invoke_out_buf(std::string v) { m_invoke_out_buf = epee::byte_slice{std::move(v)}; } int last_command() const { return m_last_command; } const std::string& last_in_buf() const { return m_last_in_buf; } @@ -127,7 +126,7 @@ namespace boost::mutex m_mutex; int m_return_code; - std::string m_invoke_out_buf; + epee::byte_slice m_invoke_out_buf; int m_last_command; std::string m_last_in_buf; diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index 882d42c02..e7e0ee247 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -64,7 +64,7 @@ namespace net_load_tests { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, test_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_connection_context& context) { //m_invoke_counter.inc(); //std::unique_lock<std::mutex> lock(m_mutex); diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index 314fcf9f2..ab6791324 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -56,13 +56,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf; + buff_out = m_invoke_out_buf.clone(); return m_return_code; } @@ -102,8 +102,7 @@ namespace int return_code() const { return m_return_code; } void return_code(int v) { m_return_code = v; } - const std::string& invoke_out_buf() const { return m_invoke_out_buf; } - void invoke_out_buf(const std::string& v) { m_invoke_out_buf = v; } + void invoke_out_buf(std::string v) { m_invoke_out_buf = epee::byte_slice{std::move(v)}; } int last_command() const { return m_last_command; } const std::string& last_in_buf() const { return m_last_in_buf; } @@ -118,7 +117,7 @@ namespace boost::mutex m_mutex; int m_return_code; - std::string m_invoke_out_buf; + epee::byte_slice m_invoke_out_buf; int m_last_command; std::string m_last_in_buf; diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index b365cad86..207c4a7dc 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -886,8 +886,6 @@ TEST(ByteStream, Empty) { epee::byte_stream stream; - EXPECT_EQ(epee::byte_stream::default_increase(), stream.increase_size()); - EXPECT_EQ(nullptr, stream.data()); EXPECT_EQ(nullptr, stream.tellp()); EXPECT_EQ(0u, stream.available()); @@ -912,43 +910,55 @@ TEST(ByteStream, Write) {0xde, 0xad, 0xbe, 0xef, 0xef}; std::vector<std::uint8_t> bytes; - epee::byte_stream stream{4}; - - EXPECT_EQ(4u, stream.increase_size()); + epee::byte_stream stream{}; stream.write({source, 3}); bytes.insert(bytes.end(), source, source + 3); EXPECT_EQ(3u, stream.size()); - EXPECT_EQ(1u, stream.available()); - EXPECT_EQ(4u, stream.capacity()); + EXPECT_LE(3u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); + const std::size_t capacity = stream.capacity(); + stream.write({source, 2}); bytes.insert(bytes.end(), source, source + 2); EXPECT_EQ(5u, stream.size()); - EXPECT_EQ(3u, stream.available()); - EXPECT_EQ(8u, stream.capacity()); + EXPECT_LE(5u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); stream.write({source, 5}); bytes.insert(bytes.end(), source, source + 5); EXPECT_EQ(10u, stream.size()); - EXPECT_EQ(2u, stream.available()); - EXPECT_EQ(12u, stream.capacity()); + EXPECT_LE(10u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); stream.write({source, 2}); bytes.insert(bytes.end(), source, source + 2); EXPECT_EQ(12u, stream.size()); - EXPECT_EQ(0u, stream.available()); - EXPECT_EQ(12u, stream.capacity()); + EXPECT_LE(12u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); stream.write({source, 5}); bytes.insert(bytes.end(), source, source + 5); EXPECT_EQ(17u, stream.size()); - EXPECT_EQ(0u, stream.available()); - EXPECT_EQ(17u, stream.capacity()); + EXPECT_LE(17u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); + + // ensure it can overflow properly + while (capacity == stream.capacity()) + { + stream.write({source, 5}); + bytes.insert(bytes.end(), source, source + 5); + } + + EXPECT_EQ(bytes.size(), stream.size()); + EXPECT_LE(bytes.size(), stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); } @@ -967,8 +977,8 @@ TEST(ByteStream, Put) } EXPECT_EQ(200u, stream.size()); - EXPECT_EQ(epee::byte_stream::default_increase() - 200, stream.available()); - EXPECT_EQ(epee::byte_stream::default_increase(), stream.capacity()); + EXPECT_LE(200u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); } @@ -981,14 +991,12 @@ TEST(ByteStream, Reserve) {0xde, 0xad, 0xbe, 0xef, 0xef}; std::vector<std::uint8_t> bytes; - epee::byte_stream stream{4}; - - EXPECT_EQ(4u, stream.increase_size()); + epee::byte_stream stream{}; stream.reserve(100); - EXPECT_EQ(100u, stream.capacity()); + EXPECT_LE(100u, stream.capacity()); EXPECT_EQ(0u, stream.size()); - EXPECT_EQ(100u, stream.available()); + EXPECT_EQ(stream.available(), stream.capacity()); for (std::size_t i = 0; i < 100 / sizeof(source); ++i) { @@ -997,8 +1005,8 @@ TEST(ByteStream, Reserve) } EXPECT_EQ(100u, stream.size()); - EXPECT_EQ(0u, stream.available()); - EXPECT_EQ(100u, stream.capacity()); + EXPECT_LE(100u, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); } @@ -1033,29 +1041,31 @@ TEST(ByteStream, Move) static constexpr const std::uint8_t source[] = {0xde, 0xad, 0xbe, 0xef, 0xef}; - epee::byte_stream stream{10}; + epee::byte_stream stream{}; stream.write(source); + const std::size_t capacity = stream.capacity(); + std::uint8_t const* const data = stream.data(); + EXPECT_LE(5u, capacity); + EXPECT_NE(nullptr, data); + epee::byte_stream stream2{std::move(stream)}; - EXPECT_EQ(10u, stream.increase_size()); EXPECT_EQ(0u, stream.size()); EXPECT_EQ(0u, stream.available()); EXPECT_EQ(0u, stream.capacity()); EXPECT_EQ(nullptr, stream.data()); EXPECT_EQ(nullptr, stream.tellp()); - EXPECT_EQ(10u, stream2.increase_size()); EXPECT_EQ(5u, stream2.size()); - EXPECT_EQ(5u, stream2.available()); - EXPECT_EQ(10u, stream2.capacity()); - EXPECT_NE(nullptr, stream2.data()); - EXPECT_NE(nullptr, stream2.tellp()); + EXPECT_EQ(capacity, stream2.capacity()); + EXPECT_EQ(capacity - 5, stream2.available()); + EXPECT_EQ(data, stream2.data()); + EXPECT_EQ(data + 5u, stream2.tellp()); EXPECT_TRUE(equal(source, byte_span{stream2.data(), stream2.size()})); stream = epee::byte_stream{}; - EXPECT_EQ(epee::byte_stream::default_increase(), stream.increase_size()); EXPECT_EQ(0u, stream.size()); EXPECT_EQ(0u, stream.available()); EXPECT_EQ(0u, stream.capacity()); @@ -1064,15 +1074,13 @@ TEST(ByteStream, Move) stream = std::move(stream2); - EXPECT_EQ(10u, stream.increase_size()); EXPECT_EQ(5u, stream.size()); - EXPECT_EQ(5u, stream.available()); - EXPECT_EQ(10u, stream.capacity()); + EXPECT_EQ(capacity, stream.capacity()); + EXPECT_EQ(capacity - 5, stream.available()); EXPECT_NE(nullptr, stream.data()); EXPECT_NE(nullptr, stream.tellp()); EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()})); - EXPECT_EQ(10u, stream2.increase_size()); EXPECT_EQ(0u, stream2.size()); EXPECT_EQ(0u, stream2.available()); EXPECT_EQ(0u, stream2.capacity()); @@ -1122,9 +1130,7 @@ TEST(ByteStream, Clear) static constexpr const std::uint8_t source[] = {0xde, 0xad, 0xbe, 0xef, 0xef}; - epee::byte_stream stream{4}; - - EXPECT_EQ(4u, stream.increase_size()); + epee::byte_stream stream{}; EXPECT_EQ(nullptr, stream.data()); EXPECT_EQ(nullptr, stream.tellp()); @@ -1146,16 +1152,17 @@ TEST(ByteStream, Clear) EXPECT_EQ(loc, stream.data()); EXPECT_EQ(loc + 3, stream.tellp()); EXPECT_EQ(3u, stream.size()); - EXPECT_EQ(1u, stream.available()); - EXPECT_EQ(4u, stream.capacity()); + EXPECT_LE(stream.size(), stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); + const std::size_t capacity = stream.capacity(); stream.clear(); EXPECT_EQ(loc, stream.data()); EXPECT_EQ(loc, stream.tellp()); EXPECT_EQ(0u, stream.size()); - EXPECT_EQ(4u, stream.available()); - EXPECT_EQ(4u, stream.capacity()); + EXPECT_EQ(capacity, stream.capacity()); + EXPECT_EQ(stream.available(), stream.capacity() - stream.size()); } TEST(ToHex, String) diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index 22638942d..2ad149afe 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -238,9 +238,9 @@ namespace return {connection, std::move(request)}; } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, cryptonote::levin::detail::p2p_context& context) override final + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, cryptonote::levin::detail::p2p_context& context) override final { - buff_out.clear(); + buff_out = nullptr; invoked_.push_back( {context.m_connection_id, command, std::string{reinterpret_cast<const char*>(in_buff.data()), in_buff.size()}} ); diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index f5aef4796..dffda5e4e 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -245,7 +245,7 @@ namespace TEST(tor_address, epee_serializev_v2) { - std::string buffer{}; + epee::byte_slice buffer{}; { test_command_tor command{MONERO_UNWRAP(net::tor_address::make(v2_onion, 10))}; EXPECT_FALSE(command.tor.is_unknown()); @@ -266,7 +266,7 @@ TEST(tor_address, epee_serializev_v2) EXPECT_EQ(0u, command.tor.port()); epee::serialization::portable_storage stg{}; - EXPECT_TRUE(stg.load_from_binary(buffer)); + EXPECT_TRUE(stg.load_from_binary(epee::to_span(buffer))); EXPECT_TRUE(command.load(stg)); } EXPECT_FALSE(command.tor.is_unknown()); @@ -277,7 +277,7 @@ TEST(tor_address, epee_serializev_v2) // make sure that exceeding max buffer doesn't destroy tor_address::_load { epee::serialization::portable_storage stg{}; - stg.load_from_binary(buffer); + stg.load_from_binary(epee::to_span(buffer)); std::string host{}; ASSERT_TRUE(stg.get_value("host", host, stg.open_section("tor", nullptr, false))); @@ -296,7 +296,7 @@ TEST(tor_address, epee_serializev_v2) TEST(tor_address, epee_serializev_v3) { - std::string buffer{}; + epee::byte_slice buffer{}; { test_command_tor command{MONERO_UNWRAP(net::tor_address::make(v3_onion, 10))}; EXPECT_FALSE(command.tor.is_unknown()); @@ -317,7 +317,7 @@ TEST(tor_address, epee_serializev_v3) EXPECT_EQ(0u, command.tor.port()); epee::serialization::portable_storage stg{}; - EXPECT_TRUE(stg.load_from_binary(buffer)); + EXPECT_TRUE(stg.load_from_binary(epee::to_span(buffer))); EXPECT_TRUE(command.load(stg)); } EXPECT_FALSE(command.tor.is_unknown()); @@ -328,7 +328,7 @@ TEST(tor_address, epee_serializev_v3) // make sure that exceeding max buffer doesn't destroy tor_address::_load { epee::serialization::portable_storage stg{}; - stg.load_from_binary(buffer); + stg.load_from_binary(epee::to_span(buffer)); std::string host{}; ASSERT_TRUE(stg.get_value("host", host, stg.open_section("tor", nullptr, false))); @@ -347,7 +347,7 @@ TEST(tor_address, epee_serializev_v3) TEST(tor_address, epee_serialize_unknown) { - std::string buffer{}; + epee::byte_slice buffer{}; { test_command_tor command{net::tor_address::unknown()}; EXPECT_TRUE(command.tor.is_unknown()); @@ -368,7 +368,7 @@ TEST(tor_address, epee_serialize_unknown) EXPECT_EQ(0u, command.tor.port()); epee::serialization::portable_storage stg{}; - EXPECT_TRUE(stg.load_from_binary(buffer)); + EXPECT_TRUE(stg.load_from_binary(epee::to_span(buffer))); EXPECT_TRUE(command.load(stg)); } EXPECT_TRUE(command.tor.is_unknown()); @@ -379,7 +379,7 @@ TEST(tor_address, epee_serialize_unknown) // make sure that exceeding max buffer doesn't destroy tor_address::_load { epee::serialization::portable_storage stg{}; - stg.load_from_binary(buffer); + stg.load_from_binary(epee::to_span(buffer)); std::string host{}; ASSERT_TRUE(stg.get_value("host", host, stg.open_section("tor", nullptr, false))); @@ -700,7 +700,7 @@ namespace TEST(i2p_address, epee_serializev_b32) { - std::string buffer{}; + epee::byte_slice buffer{}; { test_command_i2p command{MONERO_UNWRAP(net::i2p_address::make(b32_i2p, 10))}; EXPECT_FALSE(command.i2p.is_unknown()); @@ -721,7 +721,7 @@ TEST(i2p_address, epee_serializev_b32) EXPECT_EQ(0u, command.i2p.port()); epee::serialization::portable_storage stg{}; - EXPECT_TRUE(stg.load_from_binary(buffer)); + EXPECT_TRUE(stg.load_from_binary(epee::to_span(buffer))); EXPECT_TRUE(command.load(stg)); } EXPECT_FALSE(command.i2p.is_unknown()); @@ -732,7 +732,7 @@ TEST(i2p_address, epee_serializev_b32) // make sure that exceeding max buffer doesn't destroy i2p_address::_load { epee::serialization::portable_storage stg{}; - stg.load_from_binary(buffer); + stg.load_from_binary(epee::to_span(buffer)); std::string host{}; ASSERT_TRUE(stg.get_value("host", host, stg.open_section("i2p", nullptr, false))); @@ -751,7 +751,7 @@ TEST(i2p_address, epee_serializev_b32) TEST(i2p_address, epee_serialize_unknown) { - std::string buffer{}; + epee::byte_slice buffer{}; { test_command_i2p command{net::i2p_address::unknown()}; EXPECT_TRUE(command.i2p.is_unknown()); @@ -772,7 +772,7 @@ TEST(i2p_address, epee_serialize_unknown) EXPECT_EQ(0u, command.i2p.port()); epee::serialization::portable_storage stg{}; - EXPECT_TRUE(stg.load_from_binary(buffer)); + EXPECT_TRUE(stg.load_from_binary(epee::to_span(buffer))); EXPECT_TRUE(command.load(stg)); } EXPECT_TRUE(command.i2p.is_unknown()); @@ -783,7 +783,7 @@ TEST(i2p_address, epee_serialize_unknown) // make sure that exceeding max buffer doesn't destroy i2p_address::_load { epee::serialization::portable_storage stg{}; - stg.load_from_binary(buffer); + stg.load_from_binary(epee::to_span(buffer)); std::string host{}; ASSERT_TRUE(stg.get_value("host", host, stg.open_section("i2p", nullptr, false))); diff --git a/tests/unit_tests/test_protocol_pack.cpp b/tests/unit_tests/test_protocol_pack.cpp index 312f18dfe..1a4fd30f8 100644 --- a/tests/unit_tests/test_protocol_pack.cpp +++ b/tests/unit_tests/test_protocol_pack.cpp @@ -36,7 +36,7 @@ TEST(protocol_pack, protocol_pack_command) { - std::string buff; + epee::byte_slice buff; cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request r; r.start_height = 1; r.total_height = 3; @@ -47,7 +47,7 @@ TEST(protocol_pack, protocol_pack_command) ASSERT_TRUE(res); cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request r2; - res = epee::serialization::load_t_from_binary(r2, buff); + res = epee::serialization::load_t_from_binary(r2, epee::to_span(buff)); ASSERT_TRUE(res); ASSERT_TRUE(r.m_block_ids.size() == i); ASSERT_TRUE(r.start_height == 1); diff --git a/utils/health/README.md b/utils/health/README.md new file mode 100644 index 000000000..dea46280e --- /dev/null +++ b/utils/health/README.md @@ -0,0 +1,34 @@ +#Intro +This directory contains tools, which can be used for checking the health of the project, like build/run time analyzers, lints, etc. + +#Usage +Unless it's stated differently, these scripts should be called from a given source directory, where you want the checks to be performed, for instance: + +`og@ghetto:~/dev/monero$ utils/health/clang-build-time-analyzer-run.sh` + +##ClangBuildAnalyzer +`utils/health/clang-build-time-analyzer-run.sh` +The CBA helps in finding culprints of slow compilation. +On the first run, the script will complain about the missing ClangBuildAnalyzer binary and will point you to another script, which is able to clone and build the required binary. + +##clang-tidy +`utils/health/clang-tidy-run.sh` +Performs Lint checks on the source code and stores the result in the build directory. More information on the [home page](https://clang.llvm.org/extra/clang-tidy/). + + +##Valgrind checks +`utils/health/valgrind-tests.sh` +This script is able to run valgrind's callgrind, cachegrind and memcheck for a given set of executables. +It expects ONE PARAMETER, which points to a file with paths to executables and their arguments, written line by line. For example: + +``` +ls -l -h +build/tests/unit_tests/unit_tests +``` + +The `*.out` results can be interpreted with the `kcachegrind` tool. +The memcheck output is just a readable text file with a summary at the end. + +#Footer +Responsible: mj-xmr + diff --git a/utils/health/build-scripts/clang-build-time-analyzer-clone-build.sh b/utils/health/build-scripts/clang-build-time-analyzer-clone-build.sh new file mode 100755 index 000000000..de5f35a38 --- /dev/null +++ b/utils/health/build-scripts/clang-build-time-analyzer-clone-build.sh @@ -0,0 +1,54 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This script checkouts and builds ClangBuildAnalyzer. +# The result is put into bin directory + +DIR_THIS="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +TAG="v1.2.0" +PROG="ClangBuildAnalyzer" +DIR_OUT="$DIR_THIS/../bin" +DIR_BUILD="build" + +mkdir -p "$DIR_BUILD" && cd "$DIR_BUILD" +if [ ! -d "$PROG" ]; then + git clone https://github.com/aras-p/$PROG.git +fi +cd "$PROG" +git checkout "$TAG" +mkdir -p build && cd build +cmake .. +make -j`nproc` + +mkdir -p "$DIR_OUT" +cp -v "$PROG" "$DIR_OUT" +make clean # Clean the used space + diff --git a/utils/health/clang-build-time-analyzer-run.sh b/utils/health/clang-build-time-analyzer-run.sh new file mode 100755 index 000000000..fb7eeaced --- /dev/null +++ b/utils/health/clang-build-time-analyzer-run.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# ClangBuildAnalyzer is able to analyze the aggregate build time of particular headers. + +DIR_THIS="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Build variables +PROG="ClangBuildAnalyzer" +PROG_PATH="$DIR_THIS/bin/$PROG" +DIR_BUILD="build/clang-build-analyser" + +# ClangBuildAnalyzer variables +DIR_MONITORED="." +RESULT="cba-result.txt" +TRACE="cba-trace.txt" + +if [ -f "$PROG_PATH" ]; then + echo "Found: $PROG_PATH" +else + echo "Couldn't find: $PROG_PATH" + echo "Please run the below script to clone and build $PROG:" + echo "$DIR_THIS/build-scripts/clang-build-time-analyzer-clone-build.sh" + exit 1 +fi + +mkdir -p "$DIR_BUILD" && cd "$DIR_BUILD" + +cmake ../.. \ +-DCMAKE_C_COMPILER=clang \ +-DCMAKE_CXX_COMPILER=clang++ \ +-DUSE_CCACHE=OFF \ +-DUSE_COMPILATION_TIME_PROFILER=ON \ +-DBUILD_SHARED_LIBS=ON \ +-DBUILD_TESTS=ON + +make clean # Clean up, so that the trace can be regenerated from scratch +$PROG_PATH --start $DIR_MONITORED # Start monitoring +time make # Build +#time make easylogging # Quick testing: build a single target +$PROG_PATH --stop $DIR_MONITORED $TRACE # Stop and output to trace file +$PROG_PATH --analyze $TRACE | tee $RESULT # Analyze the trace, and store it in a readable format +gzip -f $TRACE # Zip the trace, because it's huge. -f overwrites the previously generated trace + +echo "" +echo "Readable result stored in: $DIR_BUILD/$RESULT" +echo "The trace (analyser's input data) in: $DIR_BUILD/$TRACE.gz" + diff --git a/utils/health/clang-tidy-run.sh b/utils/health/clang-tidy-run.sh new file mode 100755 index 000000000..6b34f6a3b --- /dev/null +++ b/utils/health/clang-tidy-run.sh @@ -0,0 +1,65 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# clang-tidy runs lint checks on C & C++ sources and headers. +# Run this script from the source directory. + +DIR_BUILD_BASE="build/clang-tidy" +RESULT_BASE="clang-tidy-result" + +function tidy_for_language() { + LANG="${1}" + DIR_BUILD="${DIR_BUILD_BASE}-${LANG}" + RESULT="${RESULT_BASE}-${LANG}.txt" + + mkdir -p "$DIR_BUILD" && pushd "$DIR_BUILD" + + cmake ../.. \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DUSE_CCACHE=ON \ + -DUSE_CLANG_TIDY_${LANG}=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTS=ON + + make clean # Clean up, so that the result can be regenerated from scratch + time make -k 2>&1 | tee "$RESULT" # Build and store the result. -k means: ignore errors + #time make -k easylogging 2>&1 | tee "$RESULT" # Quick testing: build a single target + gzip -f "$RESULT" # Zip the result, because it's huge. -f overwrites the previously generated result + + echo "" + echo "Readable result stored in: $DIR_BUILD/$RESULT.gz" + + popd +} + +tidy_for_language "C" +tidy_for_language "CXX" + |