diff options
102 files changed, 2720 insertions, 1021 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e1aade2d..7d506d817 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,10 +36,15 @@ jobs: submodules: recursive - name: remove bundled boost run: sudo rm -rf /usr/local/share/boost + - name: set apt conf + run: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev + run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler - name: build run: make -j3 @@ -51,10 +56,15 @@ jobs: submodules: recursive - name: remove bundled boost run: sudo rm -rf /usr/local/share/boost + - name: set apt conf + run: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev + run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler - name: build run: cmake -DBUILD_GUI_DEPS=ON && make -j3 @@ -67,10 +77,15 @@ jobs: submodules: recursive - name: remove bundled boost run: sudo rm -rf /usr/local/share/boost + - name: set apt conf + run: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev + run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler - name: install requests run: pip install requests - name: tests diff --git a/CMakeLists.txt b/CMakeLists.txt index 03ede0483..d9ec866e4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,8 @@ message(STATUS "CMake version ${CMAKE_VERSION}") project(monero) +include(FindCcache) # Has to be included after the project() macro, to be able to read the CXX variable. + enable_language(C ASM) function (die msg) diff --git a/Dockerfile b/Dockerfile index a6fab3823..61bbd76f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,8 +55,8 @@ RUN set -ex \ ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} # OpenSSL -ARG OPENSSL_VERSION=1.1.1b -ARG OPENSSL_HASH=5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b +ARG OPENSSL_VERSION=1.1.1g +ARG OPENSSL_HASH=ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46 RUN set -ex \ && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ diff --git a/cmake/FindCcache.cmake b/cmake/FindCcache.cmake new file mode 100644 index 000000000..fa357610b --- /dev/null +++ b/cmake/FindCcache.cmake @@ -0,0 +1,57 @@ +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# - Try to find readline include dirs and libraries +# +# Automatically finds ccache build accelerator, if it's found in PATH. +# +# Usage of this module as follows: +# +# project(monero) +# include(FindCcache) # Include AFTER the project() macro to be able to reach the CMAKE_CXX_COMPILER variable +# +# Properties modified by this module: +# +# GLOBAL PROPERTY RULE_LAUNCH_COMPILE set to ccache, when ccache found +# GLOBAL PROPERTY RULE_LAUNCH_LINK set to ccache, when ccache found + +find_program(CCACHE_FOUND ccache) +if (CCACHE_FOUND) + set(TEMP_CPP_FILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/test-program.cpp") + file(WRITE "${TEMP_CPP_FILE}" "int main() { return 0; }") + execute_process(COMMAND "${CCACHE_FOUND}" "${CMAKE_CXX_COMPILER}" "${TEMP_CPP_FILE}" RESULT_VARIABLE RET) + if (${RET} EQUAL 0) + message("found usable ccache: ${CCACHE_FOUND}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_FOUND}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CCACHE_FOUND}") + else() + message("found ccache ${CCACHE_FOUND}, but is UNUSABLE! Return code: ${RET}") + endif() +else() + message("ccache NOT found!") +endif() + diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 4b07d08ba..9d3c28465 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,6 +1,6 @@ package=openssl $(package)_version=1.0.2r -$(package)_download_path=https://www.openssl.org/source +$(package)_download_path=https://ftp.openssl.org/source/old/1.0.2 $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6 $(package)_patches=fix_arflags.patch diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h index 1fbba101e..6b79f6d92 100644 --- a/contrib/epee/include/byte_slice.h +++ b/contrib/epee/include/byte_slice.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019, The Monero Project +// Copyright (c) 2019-2020, The Monero Project // // All rights reserved. // @@ -39,10 +39,22 @@ namespace epee { struct byte_slice_data; + class byte_stream; struct release_byte_slice { - void operator()(byte_slice_data*) const noexcept; + //! For use with `zmq_message_init_data`, use second arg for buffer pointer. + static void call(void*, void* ptr) noexcept; + void operator()(byte_slice_data* ptr) const noexcept + { + call(nullptr, ptr); + } + }; + + //! Frees ref count + buffer allocated internally by `byte_buffer`. + struct release_byte_buffer + { + void operator()(std::uint8_t* buf) const noexcept; }; /*! Inspired by slices in golang. Storage is thread-safe reference counted, @@ -99,6 +111,9 @@ namespace epee //! Convert `buffer` into a slice using one allocation for shared count. explicit byte_slice(std::string&& buffer); + //! Convert `stream` into a slice with zero allocations. + explicit byte_slice(byte_stream&& stream) noexcept; + byte_slice(byte_slice&& source) noexcept; ~byte_slice() noexcept = default; @@ -140,6 +155,23 @@ namespace epee \throw std::out_of_range If `size() < end`. \return Slice starting at `data() + begin` of size `end - begin`. */ byte_slice get_slice(std::size_t begin, std::size_t end) const; + + //! \post `empty()` \return Ownership of ref-counted buffer. + std::unique_ptr<byte_slice_data, release_byte_slice> take_buffer() noexcept; }; + + //! Alias for a buffer that has space for a `byte_slice` ref count. + using byte_buffer = std::unique_ptr<std::uint8_t, release_byte_buffer>; + + /*! \return `buf` with a new size of exactly `length`. New bytes not + initialized. A `nullptr` is returned on allocation failure. */ + byte_buffer byte_buffer_resize(byte_buffer buf, std::size_t length) noexcept; + + /*! Increase `buf` of size `current` by `more` bytes. + + \throw std::range_error if `current + more` exceeds `size_t` bounds. + \return Buffer of `current + more` bytes. A `nullptr` is returned on + allocation failure. */ + byte_buffer byte_buffer_increase(byte_buffer buf, std::size_t current, std::size_t more); } // epee diff --git a/contrib/epee/include/byte_stream.h b/contrib/epee/include/byte_stream.h new file mode 100644 index 000000000..98f563ca9 --- /dev/null +++ b/contrib/epee/include/byte_stream.h @@ -0,0 +1,224 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cassert> +#include <cstdint> +#include <cstring> + +#include "byte_slice.h" +#include "span.h" + +namespace epee +{ + /*! \brief A partial drop-in replacement for `std::ostream`. + + Only a few base `std::ostream` functions are implemented - enough for + rapidjson output currently. + + Advantages over `std::stringstream` or `rapidjson::StringBuffer`: + - The internal buffer can be taken without a copy. + - The internal buffer can be given to `byte_slice` with zero + allocations for reference count. + - The internal buffer can be given to `zmq_msg_data_init` without a + copy or extra allocation. + an additional advantage over `std::stringstream`: + - Construction is significantly faster - the global `std::locale` + does not have to be acquired (global thread synchronization), and + an extra allocation for `std::stringbuf` is not needed (which is an + addition to the buffer inside of that object). */ + class byte_stream + { + byte_buffer buffer_; //! Beginning of buffer + std::uint8_t* next_write_; //! Current write position + const std::uint8_t* end_; //! End of buffer + std::size_t increase_size_; //! Minimum buffer size increase + + //! \post `requested <= available()` + void overflow(const std::size_t requested); + + //! Ensures that at least `requested` bytes are available. + void check(const std::size_t requested) + { + const std::size_t remaining = available(); + if (remaining < requested) + overflow(requested); + } + + public: + using char_type = std::uint8_t; + using Ch = char_type; + + //! \return Default minimum size increase on buffer overflow + static constexpr std::size_t default_increase() noexcept { return 4096; } + + //! Increase internal buffer by at least `byte_stream_increase` bytes. + byte_stream() noexcept + : byte_stream(default_increase()) + {} + + //! Increase internal buffer by at least `increase` bytes. + explicit byte_stream(const std::size_t increase) noexcept + : buffer_(nullptr), + next_write_(nullptr), + end_(nullptr), + increase_size_(increase) + {} + + byte_stream(byte_stream&& rhs) noexcept; + ~byte_stream() noexcept = default; + byte_stream& operator=(byte_stream&& rhs) noexcept; + + //! \return The minimum increase size on buffer overflow + std::size_t increase_size() const noexcept { return increase_size_; } + + const std::uint8_t* data() const noexcept { return buffer_.get(); } + std::uint8_t* tellp() const noexcept { return next_write_; } + std::size_t available() const noexcept { return end_ - next_write_; } + std::size_t size() const noexcept { return next_write_ - buffer_.get(); } + std::size_t capacity() const noexcept { return end_ - buffer_.get(); } + + //! Compatibility with rapidjson. + void Flush() const noexcept + {} + + /*! Reserve at least `more` bytes. + \post `size() + more <= available()`. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void reserve(const std::size_t more) + { + check(more); + } + + /*! Copy `length` bytes starting at `ptr` to end of stream. + \throw std::range_error If exceeding max size_t value. + \throw std::bad_alloc If allocation fails. */ + void write(const std::uint8_t* ptr, const std::size_t length) + { + check(length); + std::memcpy(tellp(), ptr, length); + next_write_ += length; + } + + /*! Copy `length` bytes starting at `ptr` to end of stream. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void write(const char* ptr, const std::size_t length) + { + write(reinterpret_cast<const std::uint8_t*>(ptr), length); + } + + /*! Copy `source` to end of stream. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void write(const epee::span<const std::uint8_t> source) + { + write(source.data(), source.size()); + } + + /*! Copy `source` to end of stream. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void write(const epee::span<const char> source) + { + write(source.data(), source.size()); + } + + /*! Copy `ch` to end of stream. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void put(const std::uint8_t ch) + { + check(1); + put_unsafe(ch); + } + + /*! Copy `ch` to end of stream. Provides rapidjson compatability. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void Put(const std::uint8_t ch) + { + put(ch); + } + + /*! Writes `ch` to end of stream without runtime capacity checks. Must use + `reserve` before calling this function. Primarily for use with + rapidjson, which writes characters at a time but reserves memory in + blocks. Most applications want to use `put` or `write`. */ + void put_unsafe(const std::uint8_t ch) noexcept + { + assert(1 <= available()); + *(tellp()) = ch; + ++next_write_; + } + + /*! Write `ch` to end of stream `count` times. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void put_n(const std::uint8_t ch, const std::size_t count) + { + check(count); + std::memset(tellp(), count, ch); + next_write_ += count; + } + + /*! Copy `ch` to end of stream. + \throw std::range_error if exceeding max `size_t` value. + \throw std::bad_alloc if allocation fails. */ + void push_back(const std::uint8_t ch) + { + put(ch); + } + + //! \return The internal buffer. \post `size() == capacity() == 0`. + byte_buffer take_buffer() noexcept; + }; + + //! Compatability/optimization for rapidjson. + + inline void PutReserve(byte_stream& dest, const std::size_t length) + { + dest.reserve(length); + } + + //! Compatability/optimization for rapidjson. + + inline void PutUnsafe(byte_stream& dest, const std::uint8_t ch) + { + dest.put_unsafe(ch); + } + + //! Compability/optimization for rapidjson. + inline void PutN(byte_stream& dest, const std::uint8_t ch, const std::size_t count) + { + dest.put_n(ch, count); + } +} // epee + diff --git a/contrib/epee/include/net/abstract_http_client.h b/contrib/epee/include/net/abstract_http_client.h new file mode 100644 index 000000000..787ae2667 --- /dev/null +++ b/contrib/epee/include/net/abstract_http_client.h @@ -0,0 +1,87 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <string> +#include <boost/optional/optional.hpp> +#include "http_auth.h" +#include "net/net_ssl.h" + +namespace epee +{ +namespace net_utils +{ + inline const char* get_hex_vals() + { + static constexpr const char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; + return hexVals; + } + + inline const char* get_unsave_chars() + { + //static constexpr char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&"; + static constexpr const char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&"; + return unsave_chars; + } + + bool is_unsafe(unsigned char compare_char); + std::string dec_to_hex(char num, int radix); + int get_index(const char *s, char c); + std::string hex_to_dec_2bytes(const char *s); + std::string convert(char val); + std::string conver_to_url_format(const std::string& uri); + std::string convert_from_url_format(const std::string& uri); + std::string convert_to_url_format_force_all(const std::string& uri); + +namespace http +{ + class abstract_http_client + { + public: + abstract_http_client() {} + virtual ~abstract_http_client() {} + bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + virtual void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) = 0; + virtual void set_auto_connect(bool auto_connect) = 0; + virtual bool connect(std::chrono::milliseconds timeout) = 0; + virtual bool disconnect() = 0; + virtual bool is_connected(bool *ssl = NULL) = 0; + virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; + virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; + virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; + virtual uint64_t get_bytes_sent() const = 0; + virtual uint64_t get_bytes_received() const = 0; + }; + + class http_client_factory + { + public: + virtual ~http_client_factory() {} + virtual std::unique_ptr<abstract_http_client> create() = 0; + }; +} +} +} diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index a66fb7c23..bf6589c92 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -33,6 +33,7 @@ #include <string> #include <utility> +#include "memwipe.h" #include "string_tools.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -200,6 +201,11 @@ namespace net_utils this->~http_response_info(); new(this) http_response_info(); } + + void wipe() + { + memwipe(&m_body[0], m_body.size()); + } }; } } diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 588d5f0e3..86df48f65 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -47,6 +47,7 @@ #include "string_tools.h" #include "reg_exp_definer.h" +#include "abstract_http_client.h" #include "http_base.h" #include "http_auth.h" #include "to_nonconst_iterator.h" @@ -105,140 +106,11 @@ namespace net_utils //--------------------------------------------------------------------------- - static inline const char* get_hex_vals() - { - static const char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; - return hexVals; - } - - static inline const char* get_unsave_chars() - { - //static char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&"; - static const char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&"; - return unsave_chars; - } - - static inline bool is_unsafe(unsigned char compare_char) - { - if(compare_char <= 32 || compare_char >= 123) - return true; - - const char* punsave = get_unsave_chars(); - - for(int ichar_pos = 0; 0!=punsave[ichar_pos] ;ichar_pos++) - if(compare_char == punsave[ichar_pos]) - return true; - - return false; - } - - static inline - std::string dec_to_hex(char num, int radix) - { - int temp=0; - std::string csTmp; - int num_char; - - num_char = (int) num; - if (num_char < 0) - num_char = 256 + num_char; - - while (num_char >= radix) - { - temp = num_char % radix; - num_char = (int)floor((float)num_char / (float)radix); - csTmp = get_hex_vals()[temp]; - } - - csTmp += get_hex_vals()[num_char]; - - if(csTmp.size() < 2) - { - csTmp += '0'; - } - - std::reverse(csTmp.begin(), csTmp.end()); - //_mbsrev((unsigned char*)csTmp.data()); - - return csTmp; - } - static inline int get_index(const char *s, char c) { const char *ptr = (const char*)memchr(s, c, 16); return ptr ? ptr-s : -1; } - static inline - std::string hex_to_dec_2bytes(const char *s) - { - const char *hex = get_hex_vals(); - int i0 = get_index(hex, toupper(s[0])); - int i1 = get_index(hex, toupper(s[1])); - if (i0 < 0 || i1 < 0) - return std::string("%") + std::string(1, s[0]) + std::string(1, s[1]); - return std::string(1, i0 * 16 | i1); - } - - static inline std::string convert(char val) - { - std::string csRet; - csRet += "%"; - csRet += dec_to_hex(val, 16); - return csRet; - } - static inline std::string conver_to_url_format(const std::string& uri) - { - - std::string result; - - for(size_t i = 0; i!= uri.size(); i++) - { - if(is_unsafe(uri[i])) - result += convert(uri[i]); - else - result += uri[i]; - - } - - return result; - } - static inline std::string convert_from_url_format(const std::string& uri) - { - - std::string result; - - for(size_t i = 0; i!= uri.size(); i++) - { - if(uri[i] == '%' && i + 2 < uri.size()) - { - result += hex_to_dec_2bytes(uri.c_str() + i + 1); - i += 2; - } - else - result += uri[i]; - - } - - return result; - } - - static inline std::string convert_to_url_format_force_all(const std::string& uri) - { - - std::string result; - - for(size_t i = 0; i!= uri.size(); i++) - { - result += convert(uri[i]); - } - - return result; - } - - - - - namespace http { template<typename net_client_type> - class http_simple_client_template: public i_target_handler + class http_simple_client_template : public i_target_handler, public abstract_http_client { private: enum reciev_machine_state @@ -279,7 +151,7 @@ namespace net_utils public: explicit http_simple_client_template() - : i_target_handler() + : i_target_handler(), abstract_http_client() , m_net_client() , m_host_buff() , m_port() @@ -299,26 +171,19 @@ namespace net_utils const std::string &get_host() const { return m_host_buff; }; const std::string &get_port() const { return m_port; }; - bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) - { - http::url_content parsed{}; - const bool r = parse_url(address, parsed); - CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address); - set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options)); - return true; - } + using abstract_http_client::set_server; - void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) + void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) override { CRITICAL_REGION_LOCAL(m_lock); disconnect(); m_host_buff = std::move(host); m_port = std::move(port); - m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{}; + m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{}; m_net_client.set_ssl(std::move(ssl_options)); } - void set_auto_connect(bool auto_connect) + void set_auto_connect(bool auto_connect) override { m_auto_connect = auto_connect; } @@ -330,25 +195,25 @@ namespace net_utils m_net_client.set_connector(std::move(connector)); } - bool connect(std::chrono::milliseconds timeout) + bool connect(std::chrono::milliseconds timeout) override { CRITICAL_REGION_LOCAL(m_lock); return m_net_client.connect(m_host_buff, m_port, timeout); } //--------------------------------------------------------------------------- - bool disconnect() + bool disconnect() override { CRITICAL_REGION_LOCAL(m_lock); return m_net_client.disconnect(); } //--------------------------------------------------------------------------- - bool is_connected(bool *ssl = NULL) + bool is_connected(bool *ssl = NULL) override { CRITICAL_REGION_LOCAL(m_lock); return m_net_client.is_connected(ssl); } //--------------------------------------------------------------------------- - virtual bool handle_target_data(std::string& piece_of_transfer) + virtual bool handle_target_data(std::string& piece_of_transfer) override { CRITICAL_REGION_LOCAL(m_lock); m_response_info.m_body += piece_of_transfer; @@ -361,15 +226,14 @@ namespace net_utils return true; } //--------------------------------------------------------------------------- - inline - bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) + inline bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override { CRITICAL_REGION_LOCAL(m_lock); return invoke(uri, "GET", body, timeout, ppresponse_info, additional_params); } //--------------------------------------------------------------------------- - inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) + inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override { CRITICAL_REGION_LOCAL(m_lock); if(!is_connected()) @@ -442,7 +306,7 @@ namespace net_utils return false; } //--------------------------------------------------------------------------- - inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) + inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override { CRITICAL_REGION_LOCAL(m_lock); return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params); @@ -456,16 +320,21 @@ namespace net_utils return handle_reciev(timeout); } //--------------------------------------------------------------------------- - uint64_t get_bytes_sent() const + uint64_t get_bytes_sent() const override { return m_net_client.get_bytes_sent(); } //--------------------------------------------------------------------------- - uint64_t get_bytes_received() const + uint64_t get_bytes_received() const override { return m_net_client.get_bytes_received(); } //--------------------------------------------------------------------------- + void wipe_response() + { + m_response_info.wipe(); + } + //--------------------------------------------------------------------------- private: //--------------------------------------------------------------------------- inline bool handle_reciev(std::chrono::milliseconds timeout) @@ -1016,6 +885,14 @@ namespace net_utils } }; typedef http_simple_client_template<blocked_mode_client> http_simple_client; + + class http_simple_client_factory : public http_client_factory + { + public: + std::unique_ptr<abstract_http_client> create() override { + return std::unique_ptr<epee::net_utils::http::abstract_http_client>(new epee::net_utils::http::http_simple_client()); + } + }; } } } diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index 78e02c93a..a8bc945a8 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -38,7 +38,7 @@ namespace epee namespace net_utils { template<class t_request, class t_response, class t_transport> - bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET") + bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST") { std::string req_param; if(!serialization::store_t_to_json(out_struct, req_param)) @@ -72,7 +72,7 @@ namespace epee template<class t_request, class t_response, class t_transport> - bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET") + bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST") { std::string req_param; if(!serialization::store_t_to_binary(out_struct, req_param)) @@ -101,7 +101,7 @@ namespace epee } template<class t_request, class t_response, class t_transport> - bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, epee::json_rpc::error &error_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0") + bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, epee::json_rpc::error &error_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0") { epee::json_rpc::request<t_request> req_t = AUTO_VAL_INIT(req_t); req_t.jsonrpc = "2.0"; @@ -125,14 +125,14 @@ namespace epee } template<class t_request, class t_response, class t_transport> - bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0") + bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0") { epee::json_rpc::error error_struct; return invoke_http_json_rpc(uri, method_name, out_struct, result_struct, error_struct, transport, timeout, http_method, req_id); } template<class t_command, class t_transport> - bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0") + bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0") { return invoke_http_json_rpc(uri, t_command::methodname(), out_struct, result_struct, transport, timeout, http_method, req_id); } diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index dba02f270..804bafda7 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -150,7 +150,7 @@ namespace epee }; -#define CRITICAL_REGION_LOCAL(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_LOCAL(x) {} epee::critical_region_t<decltype(x)> critical_region_var(x) #define CRITICAL_REGION_BEGIN(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)> critical_region_var(x) #define CRITICAL_REGION_LOCAL1(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var1(x) #define CRITICAL_REGION_BEGIN1(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)> critical_region_var1(x) diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 5201f59c2..ba6ad73ae 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,8 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(epee STATIC byte_slice.cpp hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp - levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp + +add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp + wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp int-util.cpp) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) diff --git a/contrib/epee/src/abstract_http_client.cpp b/contrib/epee/src/abstract_http_client.cpp new file mode 100644 index 000000000..98b5b67d9 --- /dev/null +++ b/contrib/epee/src/abstract_http_client.cpp @@ -0,0 +1,142 @@ +#include "net/abstract_http_client.h" +#include "net/http_base.h" +#include "net/net_parse_helpers.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.http" + +namespace epee +{ +namespace net_utils +{ + //---------------------------------------------------------------------------------------------------- + bool is_unsafe(unsigned char compare_char) + { + if(compare_char <= 32 || compare_char >= 123) + return true; + + const char* punsave = get_unsave_chars(); + + for(int ichar_pos = 0; 0!=punsave[ichar_pos] ;ichar_pos++) + if(compare_char == punsave[ichar_pos]) + return true; + + return false; + } + //---------------------------------------------------------------------------------------------------- + std::string dec_to_hex(char num, int radix) + { + int temp=0; + std::string csTmp; + int num_char; + + num_char = (int) num; + if (num_char < 0) + num_char = 256 + num_char; + + while (num_char >= radix) + { + temp = num_char % radix; + num_char = (int)floor((float)num_char / (float)radix); + csTmp = get_hex_vals()[temp]; + } + + csTmp += get_hex_vals()[num_char]; + + if(csTmp.size() < 2) + { + csTmp += '0'; + } + + std::reverse(csTmp.begin(), csTmp.end()); + //_mbsrev((unsigned char*)csTmp.data()); + + return csTmp; + } + //---------------------------------------------------------------------------------------------------- + int get_index(const char *s, char c) + { + const char *ptr = (const char*)memchr(s, c, 16); + return ptr ? ptr-s : -1; + } + //---------------------------------------------------------------------------------------------------- + std::string hex_to_dec_2bytes(const char *s) + { + const char *hex = get_hex_vals(); + int i0 = get_index(hex, toupper(s[0])); + int i1 = get_index(hex, toupper(s[1])); + if (i0 < 0 || i1 < 0) + return std::string("%") + std::string(1, s[0]) + std::string(1, s[1]); + return std::string(1, i0 * 16 | i1); + } + //---------------------------------------------------------------------------------------------------- + std::string convert(char val) + { + std::string csRet; + csRet += "%"; + csRet += dec_to_hex(val, 16); + return csRet; + } + //---------------------------------------------------------------------------------------------------- + std::string conver_to_url_format(const std::string& uri) + { + + std::string result; + + for(size_t i = 0; i!= uri.size(); i++) + { + if(is_unsafe(uri[i])) + result += convert(uri[i]); + else + result += uri[i]; + + } + + return result; + } + //---------------------------------------------------------------------------------------------------- + std::string convert_from_url_format(const std::string& uri) + { + + std::string result; + + for(size_t i = 0; i!= uri.size(); i++) + { + if(uri[i] == '%' && i + 2 < uri.size()) + { + result += hex_to_dec_2bytes(uri.c_str() + i + 1); + i += 2; + } + else + result += uri[i]; + + } + + return result; + } + //---------------------------------------------------------------------------------------------------- + std::string convert_to_url_format_force_all(const std::string& uri) + { + std::string result; + + for(size_t i = 0; i!= uri.size(); i++) + { + result += convert(uri[i]); + } + return result; + } + +namespace http +{ + //---------------------------------------------------------------------------------------------------- + bool epee::net_utils::http::abstract_http_client::set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options) + { + http::url_content parsed{}; + const bool r = parse_url(address, parsed); + CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address); + set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options)); + return true; + } +} +} +} diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp index 216049e5b..99c37fae3 100644 --- a/contrib/epee/src/byte_slice.cpp +++ b/contrib/epee/src/byte_slice.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019, The Monero Project +// Copyright (c) 2019-2020, The Monero Project // // All rights reserved. // @@ -34,6 +34,7 @@ #include <utility> #include "byte_slice.h" +#include "byte_stream.h" namespace epee { @@ -49,12 +50,16 @@ namespace epee std::atomic<std::size_t> ref_count; }; - void release_byte_slice::operator()(byte_slice_data* ptr) const noexcept + void release_byte_slice::call(void*, void* ptr) noexcept { - if (ptr && --(ptr->ref_count) == 0) + if (ptr) { - ptr->~byte_slice_data(); - free(ptr); + byte_slice_data* self = static_cast<byte_slice_data*>(ptr); + if (--(self->ref_count) == 0) + { + self->~byte_slice_data(); + free(self); + } } } @@ -113,6 +118,12 @@ namespace epee } } // anonymous + void release_byte_buffer::operator()(std::uint8_t* buf) const noexcept + { + if (buf) + std::free(buf - sizeof(raw_byte_slice)); + } + byte_slice::byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept : storage_(storage), portion_(portion) { @@ -159,6 +170,14 @@ namespace epee : byte_slice(adapt_buffer{}, std::move(buffer)) {} + byte_slice::byte_slice(byte_stream&& stream) noexcept + : storage_(nullptr), portion_(stream.data(), stream.size()) + { + std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice); + new (data) raw_byte_slice{}; + storage_.reset(reinterpret_cast<raw_byte_slice*>(data)); + } + byte_slice::byte_slice(byte_slice&& source) noexcept : storage_(std::move(source.storage_)), portion_(source.portion_) { @@ -206,4 +225,36 @@ namespace epee return {}; return {storage_.get(), {portion_.begin() + begin, end - begin}}; } + + std::unique_ptr<byte_slice_data, release_byte_slice> byte_slice::take_buffer() noexcept + { + std::unique_ptr<byte_slice_data, release_byte_slice> out{std::move(storage_)}; + portion_ = nullptr; + return out; + } + + byte_buffer byte_buffer_resize(byte_buffer buf, const std::size_t length) noexcept + { + if (std::numeric_limits<std::size_t>::max() - sizeof(raw_byte_slice) < length) + return nullptr; + + std::uint8_t* data = buf.get(); + if (data != nullptr) + data -= sizeof(raw_byte_slice); + + data = static_cast<std::uint8_t*>(std::realloc(data, sizeof(raw_byte_slice) + length)); + if (data == nullptr) + return nullptr; + + buf.release(); + buf.reset(data + sizeof(raw_byte_slice)); + return buf; + } + + byte_buffer byte_buffer_increase(byte_buffer buf, const std::size_t current, const std::size_t more) + { + if (std::numeric_limits<std::size_t>::max() - current < more) + throw std::range_error{"byte_buffer_increase size_t overflow"}; + return byte_buffer_resize(std::move(buf), current + more); + } } // epee diff --git a/contrib/epee/src/byte_stream.cpp b/contrib/epee/src/byte_stream.cpp new file mode 100644 index 000000000..e87d9f0bc --- /dev/null +++ b/contrib/epee/src/byte_stream.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "byte_stream.h" + +#include <algorithm> +#include <limits> +#include <utility> + +#include <iostream> + +namespace epee +{ + void byte_stream::overflow(const std::size_t requested) + { + // Recalculating `need` bytes removes at least one instruction from every + // inlined `put` call in header + + assert(available() < requested); + const std::size_t need = requested - available(); + + const std::size_t len = size(); + const std::size_t cap = capacity(); + const std::size_t increase = std::max(need, increase_size()); + + next_write_ = nullptr; + end_ = nullptr; + + buffer_ = byte_buffer_increase(std::move(buffer_), cap, increase); + if (!buffer_) + throw std::bad_alloc{}; + + next_write_ = buffer_.get() + len; + end_ = buffer_.get() + cap + increase; + } + + byte_stream::byte_stream(byte_stream&& rhs) noexcept + : buffer_(std::move(rhs.buffer_)), + next_write_(rhs.next_write_), + end_(rhs.end_), + increase_size_(rhs.increase_size_) + { + rhs.next_write_ = nullptr; + rhs.end_ = nullptr; + } + + byte_stream& byte_stream::operator=(byte_stream&& rhs) noexcept + { + if (this != std::addressof(rhs)) + { + buffer_ = std::move(rhs.buffer_); + next_write_ = rhs.next_write_; + end_ = rhs.end_; + increase_size_ = rhs.increase_size_; + rhs.next_write_ = nullptr; + rhs.end_ = nullptr; + } + return *this; + } + + byte_buffer byte_stream::take_buffer() noexcept + { + byte_buffer out{std::move(buffer_)}; + next_write_ = nullptr; + end_ = nullptr; + return out; + } +} diff --git a/contrib/epee/src/hex.cpp b/contrib/epee/src/hex.cpp index b25ce5421..b53efe6b8 100644 --- a/contrib/epee/src/hex.cpp +++ b/contrib/epee/src/hex.cpp @@ -141,7 +141,7 @@ namespace epee result.reserve(count / 2); // the data to work with (std::string is always null-terminated) - auto data = src.data(); + auto data = src.begin(); // convert a single hex character to an unsigned integer auto char_to_int = [](const char *input) { @@ -167,9 +167,9 @@ namespace epee }; // keep going until we reach the end - while (data[0] != '\0') { + while (data != src.end()) { // skip unwanted characters - if (!include(data[0])) { + if (!include(*data)) { ++data; continue; } diff --git a/external/trezor-common b/external/trezor-common -Subproject 31a0073c62738827b48d725facd376687942912 +Subproject bff7fdfe436c727982cc553bdfb29a9021b423b diff --git a/src/common/util.cpp b/src/common/util.cpp index 747235646..f1140d1d5 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -66,7 +66,6 @@ using namespace epee; #include "util.h" #include "stack_trace.h" #include "memwipe.h" -#include "cryptonote_config.h" #include "net/http_client.h" // epee::net_utils::... #include "readline_buffer.h" @@ -1074,16 +1073,33 @@ std::string get_nix_version_display_string() { if (seconds < 60) return std::to_string(seconds) + " seconds"; + std::stringstream ss; + ss << std::fixed << std::setprecision(1); if (seconds < 3600) - return std::to_string((uint64_t)(seconds / 60)) + " minutes"; + { + ss << seconds / 60.f; + return ss.str() + " minutes"; + } if (seconds < 3600 * 24) - return std::to_string((uint64_t)(seconds / 3600)) + " hours"; - if (seconds < 3600 * 24 * 30.5) - return std::to_string((uint64_t)(seconds / (3600 * 24))) + " days"; - if (seconds < 3600 * 24 * 365.25) - return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5))) + " months"; - if (seconds < 3600 * 24 * 365.25 * 100) - return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5 * 365.25))) + " years"; + { + ss << seconds / 3600.f; + return ss.str() + " hours"; + } + if (seconds < 3600 * 24 * 30.5f) + { + ss << seconds / (3600 * 24.f); + return ss.str() + " days"; + } + if (seconds < 3600 * 24 * 365.25f) + { + ss << seconds / (3600 * 24 * 30.5f); + return ss.str() + " months"; + } + if (seconds < 3600 * 24 * 365.25f * 100) + { + ss << seconds / (3600 * 24 * 365.25f); + return ss.str() + " years"; + } return "a long time"; } @@ -1279,6 +1295,86 @@ std::string get_nix_version_display_string() return lines; } + // Calculate a "sync weight" over ranges of blocks in the blockchain, suitable for + // calculating sync time estimates + uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks) + { + if (nettype != cryptonote::MAINNET) + { + // No detailed data available except for Mainnet: Give back the number of blocks + // as a very simple and non-varying block sync weight for ranges of Testnet and + // Stagenet blocks + return num_blocks; + } + + // The following is a table of average blocks sizes in bytes over the Monero mainnet + // blockchain, where the block size is averaged over ranges of 10,000 blocks + // (about 2 weeks worth of blocks each). + // The first array entry of 442 thus means "The average byte size of the blocks + // 0 .. 9,999 is 442". The info "block_size" from the "get_block_header_by_height" + // RPC call was used for calculating this. This table (and the whole mechanism + // of calculating a "sync weight") is most important when estimating times for + // syncing from scratch. Without it the fast progress through the (in comparison) + // rather small blocks in the early blockchain) would lead to vastly underestimated + // total sync times. + // It's no big problem for estimates that this table will, over time, and if not + // updated, miss larger and larger parts at the top of the blockchain, as long + // as block size averages there do not differ wildly. + // Without time-consuming tests it's hard to say how much the estimates would + // improve if one would not only take block sizes into account, but also varying + // verification times i.e. the different CPU effort needed for the different + // transaction types (pre / post RingCT, pre / post Bulletproofs). + // Testnet and Stagenet are neglected here because of their much smaller + // importance. + static const uint32_t average_block_sizes[] = + { + 442, 1211, 1445, 1763, 2272, 8217, 5603, 9999, 16358, 10805, 5290, 4362, + 4325, 5584, 4515, 5008, 4789, 5196, 7660, 3829, 6034, 2925, 3762, 2545, + 2437, 2553, 2167, 2761, 2015, 1969, 2350, 1731, 2367, 2078, 2026, 3518, + 2214, 1908, 1780, 1640, 1976, 1647, 1921, 1716, 1895, 2150, 2419, 2451, + 2147, 2327, 2251, 1644, 1750, 1481, 1570, 1524, 1562, 1668, 1386, 1494, + 1637, 1880, 1431, 1472, 1637, 1363, 1762, 1597, 1999, 1564, 1341, 1388, + 1530, 1476, 1617, 1488, 1368, 1906, 1403, 1695, 1535, 1598, 1318, 1234, + 1358, 1406, 1698, 1554, 1591, 1758, 1426, 2389, 1946, 1533, 1308, 2701, + 1525, 1653, 3580, 1889, 2913, 8164, 5154, 3762, 3356, 4360, 3589, 4844, + 4232, 3781, 3882, 5924, 10790, 7185, 7442, 8214, 8509, 7484, 6939, 7391, + 8210, 15572, 39680, 44810, 53873, 54639, 68227, 63428, 62386, 68504, + 83073, 103858, 117573, 98089, 96793, 102337, 94714, 129568, 251584, + 132026, 94579, 94516, 95722, 106495, 121824, 153983, 162338, 136608, + 137104, 109872, 91114, 84757, 96339, 74251, 94314, 143216, 155837, + 129968, 120201, 109913, 101588, 97332, 104611, 95310, 93419, 113345, + 100743, 92152, 57565, 22533, 37564, 21823, 19980, 18277, 18402, 14344, + 12142, 15842, 13677, 17631, 18294, 22270, 41422, 39296, 36688, 33512, + 33831, 27582, 22276, 27516, 27317, 25505, 24426, 20566, 23045, 26766, + 28185, 26169, 27011, + 28642 // Blocks 1,990,000 to 1,999,999 in December 2019 + }; + const uint64_t block_range_size = 10000; + + uint64_t num_block_sizes = sizeof(average_block_sizes) / sizeof(average_block_sizes[0]); + uint64_t weight = 0; + uint64_t table_index = start_block / block_range_size; + for (;;) { + if (num_blocks == 0) + { + break; + } + if (table_index >= num_block_sizes) + { + // Take all blocks beyond our table as having the size of the blocks + // in the last table entry i.e. in the most recent known block range + weight += num_blocks * average_block_sizes[num_block_sizes - 1]; + break; + } + uint64_t portion_size = std::min(num_blocks, block_range_size - start_block % block_range_size); + weight += portion_size * average_block_sizes[table_index]; + table_index++; + num_blocks -= portion_size; + start_block += portion_size; + } + return weight; + } + std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns) { std::vector<std::string> lines; diff --git a/src/common/util.h b/src/common/util.h index b794d7908..25137ab64 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -46,6 +46,7 @@ #endif #include "crypto/hash.h" +#include "cryptonote_config.h" /*! \brief Various Tools * @@ -252,4 +253,6 @@ namespace tools void clear_screen(); std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns); + + uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks); } diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 0ec992de9..8a03f28bb 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -294,6 +294,7 @@ namespace crypto { sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); if (!sc_isnonzero((const unsigned char*)sig.r.data)) goto try_again; + memwipe(&k, sizeof(k)); } bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) { @@ -390,6 +391,8 @@ namespace crypto { // sig.r = k - sig.c*r sc_mulsub(&sig.r, &sig.c, &unwrap(r), &k); + + memwipe(&k, sizeof(k)); } bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) { @@ -560,6 +563,7 @@ POP_WARNINGS random_scalar(sig[i].c); random_scalar(sig[i].r); if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) { + memwipe(&k, sizeof(k)); local_abort("invalid pubkey"); } ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r); @@ -573,6 +577,8 @@ POP_WARNINGS hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h); sc_sub(&sig[sec_index].c, &h, &sum); sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &unwrap(sec), &k); + + memwipe(&k, sizeof(k)); } bool crypto_ops::check_ring_signature(const hash &prefix_hash, const key_image &image, diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 22bc9844b..cb6d1ec91 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -126,6 +126,20 @@ namespace cryptonote namespace cryptonote { //--------------------------------------------------------------- + void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h, hw::device &hwdev) + { + hwdev.get_transaction_prefix_hash(tx,h); + } + + //--------------------------------------------------------------- + crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx, hw::device &hwdev) + { + crypto::hash h = null_hash; + get_transaction_prefix_hash(tx, h, hwdev); + return h; + } + + //--------------------------------------------------------------- void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) { std::ostringstream s; diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8ed3b0b43..d1b24d950 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -48,6 +48,8 @@ namespace epee namespace cryptonote { //--------------------------------------------------------------- + void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h, hw::device &hwdev); + crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx, hw::device &hwdev); void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 22861b974..c1f9927a9 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -263,14 +263,14 @@ namespace cryptonote m_blockchain_storage.set_enforce_dns_checkpoints(enforce_dns); } //----------------------------------------------------------------------------------------------- - bool core::update_checkpoints() + bool core::update_checkpoints(const bool skip_dns /* = false */) { if (m_nettype != MAINNET || m_disable_dns_checkpoints) return true; if (m_checkpoints_updating.test_and_set()) return true; bool res = true; - if (time(NULL) - m_last_dns_checkpoints_update >= 3600) + if (!skip_dns && time(NULL) - m_last_dns_checkpoints_update >= 3600) { res = m_blockchain_storage.update_checkpoints(m_checkpoints_path, true); m_last_dns_checkpoints_update = time(NULL); @@ -428,9 +428,9 @@ namespace cryptonote return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const + bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive_txes) const { - m_mempool.get_transaction_backlog(backlog); + m_mempool.get_transaction_backlog(backlog, include_sensitive_txes); return true; } //----------------------------------------------------------------------------------------------- @@ -669,7 +669,8 @@ namespace cryptonote // load json & DNS checkpoints, and verify them // with respect to what blocks we already have - CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); + const bool skip_dns_checkpoints = !command_line::get_arg(vm, arg_dns_checkpoints); + CHECK_AND_ASSERT_MES(update_checkpoints(skip_dns_checkpoints), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); // DNS versions checking if (check_updates_string == "disabled") @@ -1544,9 +1545,9 @@ namespace cryptonote return m_blockchain_storage.get_db().get_block_cumulative_difficulty(height); } //----------------------------------------------------------------------------------------------- - size_t core::get_pool_transactions_count() const + size_t core::get_pool_transactions_count(bool include_sensitive_txes) const { - return m_mempool.get_transactions_count(); + return m_mempool.get_transactions_count(include_sensitive_txes); } //----------------------------------------------------------------------------------------------- bool core::have_block(const crypto::hash& id) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 79a846de1..988f25ceb 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -469,10 +469,11 @@ namespace cryptonote /** * @copydoc tx_memory_pool::get_txpool_backlog + * @param include_sensitive_txes include private transactions * * @note see tx_memory_pool::get_txpool_backlog */ - bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const; + bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive_txes = false) const; /** * @copydoc tx_memory_pool::get_transactions @@ -514,10 +515,11 @@ namespace cryptonote /** * @copydoc tx_memory_pool::get_transactions_count + * @param include_sensitive_txes include private transactions * * @note see tx_memory_pool::get_transactions_count */ - size_t get_pool_transactions_count() const; + size_t get_pool_transactions_count(bool include_sensitive_txes = false) const; /** * @copydoc Blockchain::get_total_transactions @@ -697,7 +699,7 @@ namespace cryptonote * * @note see Blockchain::update_checkpoints() */ - bool update_checkpoints(); + bool update_checkpoints(const bool skip_dns = false); /** * @brief tells the daemon to wind down operations and stop running diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index b84a59698..3dd29dd1b 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -590,7 +590,7 @@ namespace cryptonote tx.vout[i].amount = 0; crypto::hash tx_prefix_hash; - get_transaction_prefix_hash(tx, tx_prefix_hash); + get_transaction_prefix_hash(tx, tx_prefix_hash, hwdev); rct::ctkeyV outSk; if (use_simple_rct) tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index c49a3dabc..469319824 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -210,7 +210,7 @@ namespace cryptonote // TODO: Investigate why not? if(!kept_by_block) { - if(have_tx_keyimges_as_spent(tx)) + if(have_tx_keyimges_as_spent(tx, id)) { mark_double_spend(tx); LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images"); @@ -253,7 +253,7 @@ namespace cryptonote meta.last_relayed_time = time(NULL); meta.relayed = relayed; meta.set_relay_method(tx_relay); - meta.double_spend_seen = have_tx_keyimges_as_spent(tx); + meta.double_spend_seen = have_tx_keyimges_as_spent(tx, id); meta.pruned = tx.pruned; meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); @@ -615,8 +615,8 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); m_blockchain.for_all_txpool_txes([this, &hashes, &txes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { - const auto relay_method = meta.get_relay_method(); - if (relay_method != relay_method::block && relay_method != relay_method::fluff) + const auto tx_relay_method = meta.get_relay_method(); + if (tx_relay_method != relay_method::block && tx_relay_method != relay_method::fluff) return true; const auto i = std::find(hashes.begin(), hashes.end(), txid); if (i == hashes.end()) @@ -1098,30 +1098,32 @@ namespace cryptonote return m_blockchain.get_db().txpool_has_tx(id, tx_category); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx) const + bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx, const crypto::hash& txid) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); for(const auto& in: tx.vin) { CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - if(have_tx_keyimg_as_spent(tokey_in.k_image)) + if(have_tx_keyimg_as_spent(tokey_in.k_image, txid)) return true; } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im) const + bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im, const crypto::hash& txid) const { CRITICAL_REGION_LOCAL(m_transactions_lock); - bool spent = false; const auto found = m_spent_key_images.find(key_im); - if (found != m_spent_key_images.end()) + if (found != m_spent_key_images.end() && !found->second.empty()) { - for (const crypto::hash& tx_hash : found->second) - spent |= m_blockchain.txpool_tx_matches_category(tx_hash, relay_category::broadcasted); + // If another tx is using the key image, always return as spent. + // See `insert_key_images`. + if (1 < found->second.size() || *(found->second.cbegin()) != txid) + return true; + return m_blockchain.txpool_tx_matches_category(txid, relay_category::broadcasted); } - return spent; + return false; } //--------------------------------------------------------------------------------- void tx_memory_pool::lock() const diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index ca0e50415..292d427e2 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -470,10 +470,11 @@ namespace cryptonote * @brief check if a transaction in the pool has a given spent key image * * @param key_im the spent key image to look for + * @param txid hash of the new transaction where `key_im` was seen. * * @return true if the spent key image is present, otherwise false */ - bool have_tx_keyimg_as_spent(const crypto::key_image& key_im) const; + bool have_tx_keyimg_as_spent(const crypto::key_image& key_im, const crypto::hash& txid) const; /** * @brief check if any spent key image in a transaction is in the pool @@ -484,10 +485,11 @@ namespace cryptonote * @note see tx_pool::have_tx_keyimg_as_spent * * @param tx the transaction to check spent key images of + * @param txid hash of `tx`. * * @return true if any spent key images are present in the pool, otherwise false */ - bool have_tx_keyimges_as_spent(const transaction& tx) const; + bool have_tx_keyimges_as_spent(const transaction& tx, const crypto::hash& txid) const; /** * @brief forget a transaction's spent key images diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index e6479b941..e2ad3727f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -170,6 +170,14 @@ namespace cryptonote size_t m_block_download_max_size; bool m_sync_pruned_blocks; + // Values for sync time estimates + boost::posix_time::ptime m_sync_start_time; + boost::posix_time::ptime m_period_start_time; + uint64_t m_sync_start_height; + uint64_t m_period_start_height; + uint64_t get_estimated_remaining_sync_seconds(uint64_t current_blockchain_height, uint64_t target_blockchain_height); + std::string get_periodic_sync_estimate(uint64_t current_blockchain_height, uint64_t target_blockchain_height); + boost::mutex m_buffer_mutex; double get_avg_block_size(); boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index ca06fe51c..f8e032fde 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -43,6 +43,7 @@ #include "profile_tools.h" #include "net/network_throttle-detail.hpp" #include "common/pruning.h" +#include "common/util.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.cn" @@ -362,7 +363,7 @@ namespace cryptonote uint64_t last_block_v1 = m_core.get_nettype() == TESTNET ? 624633 : m_core.get_nettype() == MAINNET ? 1009826 : (uint64_t)-1; uint64_t diff_v2 = max_block_height > last_block_v1 ? std::min(abs_diff, max_block_height - last_block_v1) : 0; MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", el::Color::Yellow, context << "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height - << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " + << " [Your node is " << abs_diff << " blocks (" << tools::get_human_readable_timespan((abs_diff - diff_v2) * DIFFICULTY_TARGET_V1 + diff_v2 * DIFFICULTY_TARGET_V2) << ") " << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks @@ -1212,6 +1213,55 @@ namespace cryptonote return 1; } + // Get an estimate for the remaining sync time from given current to target blockchain height, in seconds + template<class t_core> + uint64_t t_cryptonote_protocol_handler<t_core>::get_estimated_remaining_sync_seconds(uint64_t current_blockchain_height, uint64_t target_blockchain_height) + { + // The average sync speed varies so much, even averaged over quite long time periods like 10 minutes, + // that using some sliding window would be difficult to implement without often leading to bad estimates. + // The simplest strategy - always average sync speed over the maximum available interval i.e. since sync + // started at all (from "m_sync_start_time" and "m_sync_start_height") - gives already useful results + // and seems to be quite robust. Some quite special cases like "Internet connection suddenly becoming + // much faster after syncing already a long time, and staying fast" are not well supported however. + + if (target_blockchain_height <= current_blockchain_height) + { + // Syncing stuck, or other special circumstance: Avoid errors, simply give back 0 + return 0; + } + + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration sync_time = now - m_sync_start_time; + cryptonote::network_type nettype = m_core.get_nettype(); + + // Don't simply use remaining number of blocks for the estimate but "sync weight" as provided by + // "cumulative_block_sync_weight" which knows about strongly varying Monero mainnet block sizes + uint64_t synced_weight = tools::cumulative_block_sync_weight(nettype, m_sync_start_height, current_blockchain_height - m_sync_start_height); + float us_per_weight = (float)sync_time.total_microseconds() / (float)synced_weight; + uint64_t remaining_weight = tools::cumulative_block_sync_weight(nettype, current_blockchain_height, target_blockchain_height - current_blockchain_height); + float remaining_us = us_per_weight * (float)remaining_weight; + return (uint64_t)(remaining_us / 1e6); + } + + // Return a textual remaining sync time estimate, or the empty string if waiting period not yet over + template<class t_core> + std::string t_cryptonote_protocol_handler<t_core>::get_periodic_sync_estimate(uint64_t current_blockchain_height, uint64_t target_blockchain_height) + { + std::string text = ""; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + boost::posix_time::time_duration period_sync_time = now - m_period_start_time; + if (period_sync_time > boost::posix_time::minutes(2)) + { + // Period is over, time to report another estimate + uint64_t remaining_seconds = get_estimated_remaining_sync_seconds(current_blockchain_height, target_blockchain_height); + text = tools::get_human_readable_timespan(remaining_seconds); + + // Start the new period + m_period_start_time = now; + } + return text; + } + template<class t_core> int t_cryptonote_protocol_handler<t_core>::try_add_next_blocks(cryptonote_connection_context& context) { @@ -1240,6 +1290,9 @@ namespace cryptonote if (!starting) m_last_add_end_time = tools::get_tick_count(); }); + m_sync_start_time = boost::posix_time::microsec_clock::universal_time(); + m_sync_start_height = m_core.get_current_blockchain_height(); + m_period_start_time = m_sync_start_time; while (1) { @@ -1490,7 +1543,16 @@ namespace cryptonote if (completion_percent == 100) // never show 100% if not actually up to date completion_percent = 99; progress_message = " (" + std::to_string(completion_percent) + "%, " - + std::to_string(target_blockchain_height - current_blockchain_height) + " left)"; + + std::to_string(target_blockchain_height - current_blockchain_height) + " left"; + std::string time_message = get_periodic_sync_estimate(current_blockchain_height, target_blockchain_height); + if (!time_message.empty()) + { + uint64_t total_blocks_to_sync = target_blockchain_height - m_sync_start_height; + uint64_t total_blocks_synced = current_blockchain_height - m_sync_start_height; + progress_message += ", " + std::to_string(total_blocks_synced * 100 / total_blocks_to_sync) + "% of total synced"; + progress_message += ", estimated " + time_message + " left"; + } + progress_message += ")"; } const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); @@ -2228,8 +2290,26 @@ skip: bool t_cryptonote_protocol_handler<t_core>::on_connection_synchronized() { bool val_expected = false; - if(!m_core.is_within_compiled_block_hash_area(m_core.get_current_blockchain_height()) && m_synchronized.compare_exchange_strong(val_expected, true)) + uint64_t current_blockchain_height = m_core.get_current_blockchain_height(); + if(!m_core.is_within_compiled_block_hash_area(current_blockchain_height) && m_synchronized.compare_exchange_strong(val_expected, true)) { + if ((current_blockchain_height > m_sync_start_height) && (m_sync_spans_downloaded > 0)) + { + uint64_t synced_blocks = current_blockchain_height - m_sync_start_height; + // Report only after syncing an "interesting" number of blocks: + if (synced_blocks > 20) + { + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + uint64_t synced_seconds = (now - m_sync_start_time).total_seconds(); + if (synced_seconds == 0) + { + synced_seconds = 1; + } + float blocks_per_second = (1000 * synced_blocks / synced_seconds) / 1000.0f; + MGINFO_YELLOW("Synced " << synced_blocks << " blocks in " + << tools::get_human_readable_timespan(synced_seconds) << " (" << blocks_per_second << " blocks per second)"); + } + } MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL << "You are now synchronized with the network. You may now start monero-wallet-cli." << ENDL << ENDL diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 80b14d534..034d49918 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -695,11 +695,11 @@ bool t_rpc_command_executor::print_net_stats() if (m_is_rpc) { - if (!m_rpc_client->json_rpc_request(net_stats_req, net_stats_res, "get_net_stats", fail_message.c_str())) + if (!m_rpc_client->rpc_request(net_stats_req, net_stats_res, "/get_net_stats", fail_message.c_str())) { return true; } - if (!m_rpc_client->json_rpc_request(limit_req, limit_res, "get_limit", fail_message.c_str())) + if (!m_rpc_client->rpc_request(limit_req, limit_res, "/get_limit", fail_message.c_str())) { return true; } diff --git a/src/device/device.hpp b/src/device/device.hpp index 215e97eb6..ecc4849bf 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -56,6 +56,7 @@ namespace cryptonote struct subaddress_index; struct tx_destination_entry; struct keypair; + class transaction_prefix; } namespace hw { @@ -78,7 +79,7 @@ namespace hw { virtual void on_button_request(uint64_t code=0) {} virtual void on_button_pressed() {} virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; } - virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) { on_device = true; return boost::none; } virtual void on_progress(const device_progress& event) {} virtual ~i_device_callback() = default; }; @@ -203,6 +204,8 @@ namespace hw { virtual bool open_tx(crypto::secret_key &tx_key) = 0; + virtual void get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) = 0; + virtual bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) = 0; bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 57ac7c1b2..47156cbce 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -281,6 +281,10 @@ namespace hw { return true; } + void device_default::get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) { + cryptonote::get_transaction_prefix_hash(tx, h); + } + bool device_default::generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 5252d4129..64cad78b0 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -112,6 +112,7 @@ namespace hw { crypto::signature &sig) override; bool open_tx(crypto::secret_key &tx_key) override; + void get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) override; bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index eaa9f910d..222a84d3f 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -259,7 +259,7 @@ namespace hw { static int device_id = 0; - #define PROTOCOL_VERSION 2 + #define PROTOCOL_VERSION 3 #define INS_NONE 0x00 #define INS_RESET 0x02 @@ -296,6 +296,7 @@ namespace hw { #define INS_BLIND 0x78 #define INS_UNBLIND 0x7A #define INS_GEN_TXOUT_KEYS 0x7B + #define INS_PREFIX_HASH 0x7D #define INS_VALIDATE 0x7C #define INS_MLSAG 0x7E #define INS_CLOSE_TX 0x80 @@ -1414,6 +1415,81 @@ namespace hw { return true; } + void device_ledger::get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) { + AUTO_LOCK_CMD(); + + int pref_length = 0, pref_offset = 0, offset = 0; + + #ifdef DEBUG_HWDEVICE + crypto::hash h_x; + this->controle_device->get_transaction_prefix_hash(tx,h_x); + MDEBUG("get_transaction_prefix_hash [[IN]] h_x/1 "<<h_x); + #endif + + std::ostringstream s_x; + binary_archive<true> a_x(s_x); + CHECK_AND_ASSERT_THROW_MES(::serialization::serialize(a_x, const_cast<cryptonote::transaction_prefix&>(tx)), + "unable to serialize transaction prefix"); + pref_length = s_x.str().size(); + //auto pref = std::make_unique<unsigned char[]>(pref_length); + auto uprt_pref = std::unique_ptr<unsigned char[]>{ new unsigned char[pref_length] }; + unsigned char* pref = uprt_pref.get(); + memmove(pref, s_x.str().data(), pref_length); + + offset = set_command_header_noopt(INS_PREFIX_HASH,1); + pref_offset = 0; + unsigned char v; + + //version as varint + do { + v = pref[pref_offset]; + this->buffer_send[offset] = v; + offset += 1; + pref_offset += 1; + } while (v&0x80); + + //locktime as var int + do { + v = pref[pref_offset]; + this->buffer_send[offset] = v; + offset += 1; + pref_offset += 1; + } while (v&0x80); + + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange_wait_on_input(); + + //hash remains + int cnt = 0; + while (pref_offset < pref_length) { + int len; + cnt++; + offset = set_command_header(INS_PREFIX_HASH,2,cnt); + len = pref_length - pref_offset; + //options + if (len > (BUFFER_SEND_SIZE-7)) { + len = BUFFER_SEND_SIZE-7; + this->buffer_send[offset] = 0x80; + } else { + this->buffer_send[offset] = 0x00; + } + offset += 1; + //send chunk + memmove(&this->buffer_send[offset], pref+pref_offset, len); + offset += len; + pref_offset += len; + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + } + memmove(h.data, &this->buffer_recv[0], 32); + + #ifdef DEBUG_HWDEVICE + hw::ledger::check8("prefix_hash", "h", h_x.data, h.data); + #endif + } + bool device_ledger::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { AUTO_LOCK_CMD(); diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index e3e30fba8..070162cbc 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -275,6 +275,8 @@ namespace hw { bool open_tx(crypto::secret_key &tx_key) override; + void get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) override; + bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; rct::key genCommitmentMask(const rct::key &amount_key) override; diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index a77f6697f..8bde1cb75 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -101,7 +101,7 @@ namespace trezor { return device_trezor_base::disconnect(); } - void device_trezor::device_state_reset_unsafe() + void device_trezor::device_state_initialize_unsafe() { require_connected(); if (m_live_refresh_in_progress) @@ -117,7 +117,7 @@ namespace trezor { } m_live_refresh_in_progress = false; - device_trezor_base::device_state_reset_unsafe(); + device_trezor_base::device_state_initialize_unsafe(); } void device_trezor::live_refresh_thread_main() @@ -221,7 +221,7 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(!payment_id || !subaddress || subaddress->is_zero(), "Subaddress cannot be integrated"); TREZOR_AUTO_LOCK_CMD(); require_connected(); - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); auto req = std::make_shared<messages::monero::MoneroGetAddress>(); @@ -245,7 +245,7 @@ namespace trezor { const boost::optional<cryptonote::network_type> & network_type){ TREZOR_AUTO_LOCK_CMD(); require_connected(); - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); auto req = std::make_shared<messages::monero::MoneroGetWatchKey>(); @@ -274,7 +274,7 @@ namespace trezor { { TREZOR_AUTO_LOCK_CMD(); require_connected(); - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); auto req = protocol::tx::get_tx_key(tx_aux_data); @@ -294,15 +294,15 @@ namespace trezor { TREZOR_AUTO_LOCK_CMD(); require_connected(); - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req; std::vector<protocol::ki::MoneroTransferDetails> mtds; std::vector<protocol::ki::MoneroExportedKeyImage> kis; - protocol::ki::key_image_data(wallet, transfers, mtds); - protocol::ki::generate_commitment(mtds, transfers, req); + protocol::ki::key_image_data(wallet, transfers, mtds, client_version() <= 1); + protocol::ki::generate_commitment(mtds, transfers, req, client_version() <= 1); EVENT_PROGRESS(0.); this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); @@ -386,7 +386,7 @@ namespace trezor { void device_trezor::live_refresh_start_unsafe() { - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>(); @@ -492,7 +492,7 @@ namespace trezor { TREZOR_AUTO_LOCK_CMD(); require_connected(); - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); transaction_versions_check(unsigned_tx, aux_data); @@ -514,7 +514,7 @@ namespace trezor { auto & cpend = signed_tx.ptx.back(); cpend.tx = cdata.tx; cpend.dust = 0; - cpend.fee = 0; + cpend.fee = cpend.tx.rct_signatures.txnFee; cpend.dust_added_to_fee = false; cpend.change_dts = cdata.tx_data.change_dts; cpend.selected_transfers = cdata.tx_data.selected_transfers; @@ -524,6 +524,7 @@ namespace trezor { // Transaction check try { + MDEBUG("signed transaction: " << cryptonote::get_transaction_hash(cpend.tx) << ENDL << cryptonote::obj_to_json_str(cpend.tx) << ENDL); transaction_check(cdata, aux_data); } catch(const std::exception &e){ throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what()); @@ -582,7 +583,7 @@ namespace trezor { require_connected(); if (idx > 0) - device_state_reset_unsafe(); + device_state_initialize_unsafe(); require_initialized(); EVENT_PROGRESS(0, 1, 1); @@ -670,28 +671,42 @@ namespace trezor { #undef EVENT_PROGRESS } - void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data) + unsigned device_trezor::client_version() { auto trezor_version = get_version(); - unsigned client_version = 1; // default client version for tx - if (trezor_version <= pack_version(2, 0, 10)){ - client_version = 0; + throw exc::TrezorException("Trezor firmware 2.0.10 and lower are not supported. Please update."); } + // default client version, higher versions check will be added + unsigned client_version = 1; + +#ifdef WITH_TREZOR_DEBUGGING + // Override client version for tests + const char *env_trezor_client_version = nullptr; + if ((env_trezor_client_version = getenv("TREZOR_CLIENT_VERSION")) != nullptr){ + auto succ = epee::string_tools::get_xtype_from_string(client_version, env_trezor_client_version); + if (succ){ + MINFO("Trezor client version overriden by TREZOR_CLIENT_VERSION to: " << client_version); + } + } +#endif + return client_version; + } + + void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data) + { + unsigned cversion = client_version(); + if (aux_data.client_version){ auto wanted_client_version = aux_data.client_version.get(); - if (wanted_client_version > client_version){ - throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update."); + if (wanted_client_version > cversion){ + throw exc::TrezorException("Trezor has too old firmware version. Please update."); } else { - client_version = wanted_client_version; + cversion = wanted_client_version; } } - aux_data.client_version = client_version; - - if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){ - throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update."); - } + aux_data.client_version = cversion; } void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg) diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index a26a42788..f558b7b19 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -67,10 +67,11 @@ namespace trezor { bool m_live_refresh_enabled; size_t m_num_transations_to_sign; + unsigned client_version(); void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data); void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg); void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); - void device_state_reset_unsafe() override; + void device_state_initialize_unsafe() override; void live_refresh_start_unsafe(); void live_refresh_finish_unsafe(); void live_refresh_thread_main(); diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index b7adf433d..354ae6691 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -28,6 +28,7 @@ // #include "device_trezor_base.hpp" +#include "memwipe.h" #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/regex.hpp> @@ -151,7 +152,7 @@ namespace trezor { bool device_trezor_base::disconnect() { TREZOR_AUTO_LOCK_DEVICE(); - m_device_state.clear(); + m_device_session_id.clear(); m_features.reset(); if (m_transport){ @@ -292,8 +293,8 @@ namespace trezor { case messages::MessageType_PassphraseRequest: on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get())); return true; - case messages::MessageType_PassphraseStateRequest: - on_passphrase_state_request(input, dynamic_cast<const messages::common::PassphraseStateRequest*>(input.m_msg.get())); + case messages::MessageType_Deprecated_PassphraseStateRequest: + on_passphrase_state_request(input, dynamic_cast<const messages::common::Deprecated_PassphraseStateRequest*>(input.m_msg.get())); return true; case messages::MessageType_PinMatrixRequest: on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get())); @@ -361,23 +362,34 @@ namespace trezor { return false; } - void device_trezor_base::device_state_reset_unsafe() + void device_trezor_base::device_state_initialize_unsafe() { require_connected(); + std::string tmp_session_id; auto initMsg = std::make_shared<messages::management::Initialize>(); + const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { + memwipe(&tmp_session_id[0], tmp_session_id.size()); + }); - if(!m_device_state.empty()) { - initMsg->set_allocated_state(&m_device_state); + if(!m_device_session_id.empty()) { + tmp_session_id.assign(m_device_session_id.data(), m_device_session_id.size()); + initMsg->set_allocated_session_id(&tmp_session_id); } m_features = this->client_exchange<messages::management::Features>(initMsg); - initMsg->release_state(); + if (m_features->has_session_id()){ + m_device_session_id = m_features->session_id(); + } else { + m_device_session_id.clear(); + } + + initMsg->release_session_id(); } void device_trezor_base::device_state_reset() { TREZOR_AUTO_LOCK_CMD(); - device_state_reset_unsafe(); + device_state_initialize_unsafe(); } #ifdef WITH_TREZOR_DEBUGGING @@ -441,48 +453,89 @@ namespace trezor { pin = m_pin; } - // TODO: remove PIN from memory + std::string pin_field; messages::common::PinMatrixAck m; if (pin) { - m.set_pin(pin.get().data(), pin.get().size()); + pin_field.assign(pin->data(), pin->size()); + m.set_allocated_pin(&pin_field); } + + const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { + m.release_pin(); + if (!pin_field.empty()){ + memwipe(&pin_field[0], pin_field.size()); + } + }); + resp = call_raw(&m); } void device_trezor_base::on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg) { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - MDEBUG("on_passhprase_request, on device: " << msg->on_device()); - boost::optional<epee::wipeable_string> passphrase; - TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device()); + MDEBUG("on_passhprase_request"); - if (!passphrase && m_passphrase){ - passphrase = m_passphrase; + // Backward compatibility, migration clause. + if (msg->has__on_device() && msg->_on_device()){ + messages::common::PassphraseAck m; + resp = call_raw(&m); + return; } - m_passphrase = boost::none; + bool on_device = true; + if (msg->has__on_device() && !msg->_on_device()){ + on_device = false; // do not enter on device, old devices. + } - messages::common::PassphraseAck m; - if (!msg->on_device() && passphrase){ - // TODO: remove passphrase from memory - m.set_passphrase(passphrase.get().data(), passphrase.get().size()); + if (on_device && m_features && m_features->capabilities_size() > 0){ + on_device = false; + for (auto it = m_features->capabilities().begin(); it != m_features->capabilities().end(); it++) { + if (*it == messages::management::Features::Capability_PassphraseEntry){ + on_device = true; + } + } } - if (!m_device_state.empty()){ - m.set_allocated_state(&m_device_state); + boost::optional<epee::wipeable_string> passphrase; + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + + std::string passphrase_field; + messages::common::PassphraseAck m; + m.set_on_device(on_device); + if (!on_device) { + if (!passphrase && m_passphrase) { + passphrase = m_passphrase; + } + + if (m_passphrase) { + m_passphrase = boost::none; + } + + if (passphrase) { + passphrase_field.assign(passphrase->data(), passphrase->size()); + m.set_allocated_passphrase(&passphrase_field); + } } + const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { + m.release_passphrase(); + if (!passphrase_field.empty()){ + memwipe(&passphrase_field[0], passphrase_field.size()); + } + }); + resp = call_raw(&m); - m.release_state(); } - void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg) + void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg) { MDEBUG("on_passhprase_state_request"); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - m_device_state = msg->state(); - messages::common::PassphraseStateAck m; + if (msg->has_state()) { + m_device_session_id = msg->state(); + } + messages::common::Deprecated_PassphraseStateAck m; resp = call_raw(&m); } @@ -510,7 +563,7 @@ namespace trezor { } auto msg = std::make_shared<messages::management::LoadDevice>(); - msg->set_mnemonic(mnemonic); + msg->add_mnemonics(mnemonic); msg->set_pin(pin); msg->set_passphrase_protection(passphrase_protection); msg->set_label(label); @@ -535,7 +588,8 @@ namespace trezor { return boost::none; } - boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) { + boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool & on_device) { + on_device = true; return boost::none; } diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index c106d2099..62b2c2bf4 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -70,7 +70,7 @@ namespace trezor { void on_button_request(uint64_t code=0) override; boost::optional<epee::wipeable_string> on_pin_request() override; - boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) override; void on_passphrase_state_request(const std::string &state); void on_disconnect(); protected: @@ -94,7 +94,7 @@ namespace trezor { std::string m_full_name; std::vector<unsigned int> m_wallet_deriv_path; - std::string m_device_state; // returned after passphrase entry, session + epee::wipeable_string m_device_session_id; // returned after passphrase entry, session std::shared_ptr<messages::management::Features> m_features; // features from the last device reset boost::optional<epee::wipeable_string> m_pin; boost::optional<epee::wipeable_string> m_passphrase; @@ -117,7 +117,7 @@ namespace trezor { void require_initialized() const; void call_ping_unsafe(); void test_ping(); - virtual void device_state_reset_unsafe(); + virtual void device_state_initialize_unsafe(); void ensure_derivation_path() noexcept; // Communication methods @@ -315,7 +315,7 @@ namespace trezor { void on_button_pressed(); void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg); - void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg); + void on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg); #ifdef WITH_TREZOR_DEBUGGING void set_debug(bool debug){ diff --git a/src/device_trezor/trezor/debug_link.cpp b/src/device_trezor/trezor/debug_link.cpp index c7ee59afe..102d1f966 100644 --- a/src/device_trezor/trezor/debug_link.cpp +++ b/src/device_trezor/trezor/debug_link.cpp @@ -71,9 +71,9 @@ namespace trezor{ call(decision, boost::none, true); } - void DebugLink::input_swipe(bool swipe){ + void DebugLink::input_swipe(messages::debug::DebugLinkDecision_DebugSwipeDirection direction){ messages::debug::DebugLinkDecision decision; - decision.set_up_down(swipe); + decision.set_swipe(direction); call(decision, boost::none, true); } diff --git a/src/device_trezor/trezor/debug_link.hpp b/src/device_trezor/trezor/debug_link.hpp index adf5f1d8f..a5f05ea94 100644 --- a/src/device_trezor/trezor/debug_link.hpp +++ b/src/device_trezor/trezor/debug_link.hpp @@ -49,7 +49,7 @@ namespace trezor { std::shared_ptr<messages::debug::DebugLinkState> state(); void input_word(const std::string & word); void input_button(bool button); - void input_swipe(bool swipe); + void input_swipe(messages::debug::DebugLinkDecision_DebugSwipeDirection direction); void press_yes() { input_button(true); } void press_no() { input_button(false); } void stop(); diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 61e51be14..2a7783497 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -145,7 +145,8 @@ namespace ki { bool key_image_data(wallet_shim * wallet, const std::vector<tools::wallet2::transfer_details> & transfers, - std::vector<MoneroTransferDetails> & res) + std::vector<MoneroTransferDetails> & res, + bool need_all_additionals) { for(auto & td : transfers){ ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td); @@ -157,8 +158,14 @@ namespace ki { cres.set_out_key(key_to_string(boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key)); cres.set_tx_pub_key(key_to_string(tx_pub_key)); cres.set_internal_output_index(td.m_internal_output_index); - for(auto & aux : additional_tx_pub_keys){ - cres.add_additional_tx_pub_keys(key_to_string(aux)); + cres.set_sub_addr_major(td.m_subaddr_index.major); + cres.set_sub_addr_minor(td.m_subaddr_index.minor); + if (need_all_additionals) { + for (auto &aux : additional_tx_pub_keys) { + cres.add_additional_tx_pub_keys(key_to_string(aux)); + } + } else if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) { + cres.add_additional_tx_pub_keys(key_to_string(additional_tx_pub_keys[td.m_internal_output_index])); } } @@ -188,7 +195,8 @@ namespace ki { void generate_commitment(std::vector<MoneroTransferDetails> & mtds, const std::vector<tools::wallet2::transfer_details> & transfers, - std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req) + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req, + bool need_subaddr_indices) { req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); @@ -213,11 +221,13 @@ namespace ki { st.insert(cur.m_subaddr_index.minor); } - for (auto& x: sub_indices){ - auto subs = req->add_subs(); - subs->set_account(x.first); - for(auto minor : x.second){ - subs->add_minor_indices(minor); + if (need_subaddr_indices) { + for (auto &x: sub_indices) { + auto subs = req->add_subs(); + subs->set_account(x.first); + for (auto minor : x.second) { + subs->add_minor_indices(minor); + } } } } @@ -283,26 +293,6 @@ namespace tx { translate_address(dst->mutable_addr(), &(src->addr)); } - void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){ - for(auto & cur : src->outputs){ - auto out = dst->add_outputs(); - out->set_idx(cur.first); - translate_rct_key(out->mutable_key(), &(cur.second)); - } - - dst->set_real_output(src->real_output); - dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key)); - for(auto & cur : src->real_out_additional_tx_keys){ - dst->add_real_out_additional_tx_keys(key_to_string(cur)); - } - - dst->set_real_output_in_tx_index(src->real_output_in_tx_index); - dst->set_amount(src->amount); - dst->set_rct(src->rct); - dst->set_mask(key_to_string(src->mask)); - translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki)); - } - void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){ dst->set_k(key_to_string(src->k)); dst->set_l(key_to_string(src->L)); @@ -369,6 +359,31 @@ namespace tx { return res; } + std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv) + { + // master-key-32B || domain-sep-12B || index-4B + uint8_t hash[32] = {0}; + KECCAK_CTX ctx; + std::string sep = is_iv ? "sig-iv" : "sig-key"; + std::string idx_data = tools::get_varint_data(idx); + if (idx_data.size() > 4){ + throw std::invalid_argument("index is too big"); + } + + keccak_init(&ctx); + keccak_update(&ctx, (const uint8_t *) master_key.data(), master_key.size()); + keccak_update(&ctx, (const uint8_t *) sep.data(), sep.size()); + keccak_update(&ctx, hash, 12 - sep.size()); + keccak_update(&ctx, (const uint8_t *) idx_data.data(), idx_data.size()); + if (idx_data.size() < 4) { + keccak_update(&ctx, hash, 4 - idx_data.size()); + } + + keccak_finish(&ctx, hash); + keccak(hash, sizeof(hash), hash, sizeof(hash)); + return std::string((const char*) hash, 32); + } + TData::TData() { rsig_type = 0; bp_version = 0; @@ -383,7 +398,7 @@ namespace tx { m_unsigned_tx = unsigned_tx; m_aux_data = aux_data; m_tx_idx = tx_idx; - m_ct.tx_data = cur_tx(); + m_ct.tx_data = cur_src_tx(); m_multisig = false; m_client_version = 1; } @@ -451,6 +466,41 @@ namespace tx { } } + void Signer::set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys, bool need_ring_indices){ + const cryptonote::tx_source_entry & src = cur_tx().sources[idx]; + const tools::wallet2::transfer_details & transfer = get_source_transfer(idx); + + dst->set_real_output(src.real_output); + for(size_t i = 0; i < src.outputs.size(); ++i){ + auto & cur = src.outputs[i]; + auto out = dst->add_outputs(); + + if (i == src.real_output || need_ring_indices || client_version() <= 1) { + out->set_idx(cur.first); + } + if (i == src.real_output || need_ring_keys || client_version() <= 1) { + translate_rct_key(out->mutable_key(), &(cur.second)); + } + } + + dst->set_real_out_tx_key(key_to_string(src.real_out_tx_key)); + dst->set_real_output_in_tx_index(src.real_output_in_tx_index); + + if (client_version() <= 1) { + for (auto &cur : src.real_out_additional_tx_keys) { + dst->add_real_out_additional_tx_keys(key_to_string(cur)); + } + } else if (!src.real_out_additional_tx_keys.empty()) { + dst->add_real_out_additional_tx_keys(key_to_string(src.real_out_additional_tx_keys.at(src.real_output_in_tx_index))); + } + + dst->set_amount(src.amount); + dst->set_rct(src.rct); + dst->set_mask(key_to_string(src.mask)); + translate_klrki(dst->mutable_multisig_klrki(), &(src.multisig_kLRki)); + dst->set_subaddr_minor(transfer.m_subaddr_index.minor); + } + void Signer::compute_integrated_indices(TsxData * tsx_data){ if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ return; @@ -492,6 +542,7 @@ namespace tx { // extract payment ID from construction data auto & tsx_data = m_ct.tsx_data; auto & tx = cur_tx(); + const size_t input_size = tx.sources.size(); m_ct.tx.version = 2; m_ct.tx.unlock_time = tx.unlock_time; @@ -500,12 +551,20 @@ namespace tx { tsx_data.set_version(1); tsx_data.set_client_version(client_version()); tsx_data.set_unlock_time(tx.unlock_time); - tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); + tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(input_size)); tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); tsx_data.set_account(tx.subaddr_account); tsx_data.set_monero_version(std::string(MONERO_VERSION) + "|" + MONERO_VERSION_TAG); tsx_data.set_hard_fork(m_aux_data->hard_fork ? m_aux_data->hard_fork.get() : 0); - assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); + + if (client_version() <= 1){ + assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); + } + + // TODO: use HF_VERSION_CLSAG after CLSAG is merged + if (tsx_data.hard_fork() >= 13){ + throw exc::ProtocolException("CLSAG is not yet implemented"); + } // Rsig decision auto rsig_data = tsx_data.mutable_rsig_data(); @@ -525,6 +584,11 @@ namespace tx { translate_dst_entry(dst, &cur); } + m_ct.source_permutation.clear(); + for (size_t n = 0; n < input_size; ++n){ + m_ct.source_permutation.push_back(n); + } + compute_integrated_indices(&tsx_data); int64_t fee = 0; @@ -559,7 +623,7 @@ namespace tx { CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index"); m_ct.cur_input_idx = idx; auto res = std::make_shared<messages::monero::MoneroTransactionSetInputRequest>(); - translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx])); + set_tx_input(res->mutable_src_entr(), idx, false, true); return res; } @@ -582,11 +646,6 @@ namespace tx { void Signer::sort_ki(){ const size_t input_size = cur_tx().sources.size(); - m_ct.source_permutation.clear(); - for (size_t n = 0; n < input_size; ++n){ - m_ct.source_permutation.push_back(n); - } - CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size"); std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) { const cryptonote::txin_to_key &tk0 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i0]); @@ -614,6 +673,9 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ sort_ki(); + if (client_version() >= 2){ + return nullptr; + } auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); @@ -634,17 +696,10 @@ namespace tx { auto tx = m_ct.tx_data; auto res = std::make_shared<messages::monero::MoneroTransactionInputViniRequest>(); auto & vini = m_ct.tx.vin[idx]; - translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); + set_tx_input(res->mutable_src_entr(), idx, false, false); res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); - - if (client_version() == 0) { - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); - res->set_pseudo_out(m_ct.pseudo_outs[idx]); - res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); - } - + res->set_orig_idx(m_ct.source_permutation[idx]); return res; } @@ -657,31 +712,6 @@ namespace tx { } void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ - if (client_version() > 0 || !is_offloading()){ - return; - } - - // If offloading, expect rsig configuration. - if (!ack->has_rsig_data()){ - throw exc::ProtocolException("Rsig offloading requires rsig param"); - } - - auto & rsig_data = ack->rsig_data(); - if (!rsig_data.has_mask()){ - throw exc::ProtocolException("Gamma masks not present in offloaded version"); - } - - auto & mask = rsig_data.mask(); - if (mask.size() != 32 * num_outputs()){ - throw exc::ProtocolException("Invalid number of gamma masks"); - } - - m_ct.rsig_gamma.reserve(num_outputs()); - for(size_t c=0; c < num_outputs(); ++c){ - rct::key cmask{}; - memcpy(cmask.bytes, mask.data() + c * 32, 32); - m_ct.rsig_gamma.emplace_back(cmask); - } } std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ @@ -696,15 +726,6 @@ namespace tx { auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; translate_dst_entry(res->mutable_dst_entr(), &cur_dst); res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); - - // Range sig offloading to the host - // ClientV0 sends offloaded BP with the last message in the batch. - // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks. - if (client_version() == 0 && is_offloading() && should_compute_bp_now()) { - auto rsig_data = res->mutable_rsig_data(); - compute_bproof(*rsig_data); - } - return res; } @@ -814,7 +835,7 @@ namespace tx { } std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){ - if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){ + if (!is_offloading() || !should_compute_bp_now()){ return nullptr; } @@ -917,11 +938,12 @@ namespace tx { CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index"); auto res = std::make_shared<messages::monero::MoneroTransactionSignInputRequest>(); - translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx])); + set_tx_input(res->mutable_src_entr(), idx, true, true); res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx])); res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); res->set_pseudo_out_alpha(m_ct.alphas[idx]); res->set_spend_key(m_ct.spend_encs[idx]); + res->set_orig_idx(m_ct.source_permutation[idx]); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); @@ -931,10 +953,7 @@ namespace tx { } void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){ - rct::mgSig mg; - if (!cn_deserialize(ack->signature(), mg)){ - throw exc::ProtocolException("Cannot deserialize mg[i]"); - } + m_ct.signatures.push_back(ack->signature()); // Sync updated pseudo_outputs, client_version>=1, HF10+ if (client_version() >= 1 && ack->has_pseudo_out()){ @@ -948,12 +967,9 @@ namespace tx { string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); } } - - m_ct.rv->p.MGs.push_back(mg); } std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> Signer::step_final(){ - m_ct.tx.rct_signatures = *(m_ct.rv); return std::make_shared<messages::monero::MoneroTransactionFinalRequest>(); } @@ -976,6 +992,42 @@ namespace tx { m_ct.enc_salt1 = ack->salt(); m_ct.enc_salt2 = ack->rand_mult(); m_ct.enc_keys = ack->tx_enc_keys(); + + // Opening the sealed signatures + if (client_version() >= 3){ + if(!ack->has_opening_key()){ + throw exc::ProtocolException("Client version 3+ requires sealed signatures"); + } + + for(size_t i = 0; i < m_ct.signatures.size(); ++i){ + CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size"); + std::string nonce = compute_sealing_key(ack->opening_key(), i, true); + std::string key = compute_sealing_key(ack->opening_key(), i, false); + size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[plen]); + uint8_t * buff = plaintext.get(); + + protocol::crypto::chacha::decrypt( + m_ct.signatures[i].data(), + m_ct.signatures[i].size(), + reinterpret_cast<const uint8_t *>(key.data()), + reinterpret_cast<const uint8_t *>(nonce.data()), + reinterpret_cast<char *>(buff), &plen); + m_ct.signatures[i].assign(reinterpret_cast<const char *>(buff), plen); + } + } + + // CLSAG support comes here once it is merged to the Monero + m_ct.rv->p.MGs.reserve(m_ct.signatures.size()); + for(size_t i = 0; i < m_ct.signatures.size(); ++i) { + rct::mgSig mg; + if (!cn_deserialize(m_ct.signatures[i], mg)) { + throw exc::ProtocolException("Cannot deserialize mg[i]"); + } + m_ct.rv->p.MGs.push_back(mg); + } + + m_ct.tx.rct_signatures = *(m_ct.rv); } std::string Signer::store_tx_aux_info(){ diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index f58bf1039..8fc5eba97 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -118,7 +118,8 @@ namespace ki { */ bool key_image_data(wallet_shim * wallet, const std::vector<tools::wallet2::transfer_details> & transfers, - std::vector<MoneroTransferDetails> & res); + std::vector<MoneroTransferDetails> & res, + bool need_all_additionals=false); /** * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync. @@ -130,7 +131,8 @@ namespace ki { */ void generate_commitment(std::vector<MoneroTransferDetails> & mtds, const std::vector<tools::wallet2::transfer_details> & transfers, - std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req, + bool need_subaddr_indices=false); /** * Processes Live refresh step response, parses KI, checks the signature @@ -158,13 +160,13 @@ namespace tx { void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src); void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src); - void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src); void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src); void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src); std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt); + std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv=false); typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; @@ -198,6 +200,7 @@ namespace tx { std::vector<std::string> pseudo_outs_hmac; std::vector<std::string> couts; std::vector<std::string> couts_dec; + std::vector<std::string> signatures; std::vector<rct::key> rsig_gamma; std::string tx_prefix_hash; std::string enc_salt1; @@ -221,16 +224,33 @@ namespace tx { unsigned m_client_version; bool m_multisig; - const tx_construction_data & cur_tx(){ + const tx_construction_data & cur_src_tx() const { CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index"); return m_unsigned_tx->txes[m_tx_idx]; } + const tx_construction_data & cur_tx() const { + return m_ct.tx_data; + } + + const tools::wallet2::transfer_details & get_transfer(size_t idx) const { + CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index"); + return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first]; + } + + const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const { + const auto & sel_transfers = cur_tx().selected_transfers; + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.source_permutation.size(), "Invalid source index - permutation"); + CHECK_AND_ASSERT_THROW_MES(m_ct.source_permutation[idx] < sel_transfers.size(), "Invalid source index"); + return get_transfer(sel_transfers.at(m_ct.source_permutation[idx])); + } + void extract_payment_id(); void compute_integrated_indices(TsxData * tsx_data); bool should_compute_bp_now() const; void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data); void process_bproof(rct::Bulletproof & bproof); + void set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys=false, bool need_ring_indices=false); public: Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index 59b281f13..52bee6c6c 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -56,6 +56,11 @@ namespace trezor{ return true; } + bool t_serialize(const epee::wipeable_string & in, std::string & out){ + out.assign(in.data(), in.size()); + return true; + } + bool t_serialize(const json_val & in, std::string & out){ rapidjson::StringBuffer sb; rapidjson::Writer<rapidjson::StringBuffer> writer(sb); @@ -75,6 +80,11 @@ namespace trezor{ return true; } + bool t_deserialize(std::string & in, epee::wipeable_string & out){ + out = epee::wipeable_string(in); + return true; + } + bool t_deserialize(const std::string & in, json & out){ if (out.Parse(in.c_str()).HasParseError()) { throw exc::CommunicationException("JSON parse error"); @@ -192,61 +202,69 @@ namespace trezor{ const auto msg_size = message_size(req); const auto buff_size = serialize_message_buffer_size(msg_size) + 2; - std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); - uint8_t * req_buff_raw = req_buff.get(); + epee::wipeable_string req_buff; + epee::wipeable_string chunk_buff; + + req_buff.resize(buff_size); + chunk_buff.resize(REPLEN); + + uint8_t * req_buff_raw = reinterpret_cast<uint8_t *>(req_buff.data()); + uint8_t * chunk_buff_raw = reinterpret_cast<uint8_t *>(chunk_buff.data()); + req_buff_raw[0] = '#'; req_buff_raw[1] = '#'; serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2); size_t offset = 0; - uint8_t chunk_buff[REPLEN]; // Chunk by chunk upload while(offset < buff_size){ auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1)); - chunk_buff[0] = '?'; - memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy); + chunk_buff_raw[0] = '?'; + memcpy(chunk_buff_raw + 1, req_buff_raw + offset, to_copy); // Pad with zeros if (to_copy < REPLEN - 1){ - memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy); + memset(chunk_buff_raw + 1 + to_copy, 0, REPLEN - 1 - to_copy); } - transport.write_chunk(chunk_buff, REPLEN); + transport.write_chunk(chunk_buff_raw, REPLEN); offset += REPLEN - 1; } } void ProtocolV1::read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type){ - char chunk[REPLEN]; + epee::wipeable_string chunk_buff; + chunk_buff.resize(REPLEN); + char * chunk_buff_raw = chunk_buff.data(); // Initial chunk read - size_t nread = transport.read_chunk(chunk, REPLEN); + size_t nread = transport.read_chunk(chunk_buff_raw, REPLEN); if (nread != REPLEN){ throw exc::CommunicationException("Read chunk has invalid size"); } - if (strncmp(chunk, "?##", 3) != 0){ + if (memcmp(chunk_buff_raw, "?##", 3) != 0){ throw exc::CommunicationException("Malformed chunk"); } uint16_t tag; uint32_t len; nread -= 3 + 6; - deserialize_message_header(chunk + 3, tag, len); + deserialize_message_header(chunk_buff_raw + 3, tag, len); - std::string data_acc(chunk + 3 + 6, nread); + epee::wipeable_string data_acc(chunk_buff_raw + 3 + 6, nread); data_acc.reserve(len); while(nread < len){ - const size_t cur = transport.read_chunk(chunk, REPLEN); - if (chunk[0] != '?'){ + const size_t cur = transport.read_chunk(chunk_buff_raw, REPLEN); + if (chunk_buff_raw[0] != '?'){ throw exc::CommunicationException("Chunk malformed"); } - data_acc.append(chunk + 1, cur - 1); + data_acc.append(chunk_buff_raw + 1, cur - 1); nread += cur - 1; } @@ -259,7 +277,7 @@ namespace trezor{ } std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(tag)); - if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){ + if (!msg_wrap->ParseFromArray(data_acc.data(), len)){ throw exc::CommunicationException("Message could not be parsed"); } @@ -426,15 +444,16 @@ namespace trezor{ const auto msg_size = message_size(req); const auto buff_size = serialize_message_buffer_size(msg_size); + epee::wipeable_string req_buff; + req_buff.resize(buff_size); - std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); - uint8_t * req_buff_raw = req_buff.get(); + uint8_t * req_buff_raw = reinterpret_cast<uint8_t *>(req_buff.data()); serialize_message(req, msg_size, req_buff_raw, buff_size); std::string uri = "/call/" + m_session.get(); - std::string req_hex = epee::to_hex::string(epee::span<const std::uint8_t>(req_buff_raw, buff_size)); - std::string res_hex; + epee::wipeable_string res_hex; + epee::wipeable_string req_hex = epee::to_hex::wipeable_string(epee::span<const std::uint8_t>(req_buff_raw, buff_size)); bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client); if (!req_status){ @@ -449,15 +468,15 @@ namespace trezor{ throw exc::CommunicationException("Could not read, no response stored"); } - std::string bin_data; - if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){ + boost::optional<epee::wipeable_string> bin_data = m_response->parse_hexstr(); + if (!bin_data){ throw exc::CommunicationException("Response is not well hexcoded"); } uint16_t msg_tag; uint32_t msg_len; - deserialize_message_header(bin_data.c_str(), msg_tag, msg_len); - if (bin_data.size() != msg_len + 6){ + deserialize_message_header(bin_data->data(), msg_tag, msg_len); + if (bin_data->size() != msg_len + 6){ throw exc::CommunicationException("Response is not well hexcoded"); } @@ -466,7 +485,7 @@ namespace trezor{ } std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag)); - if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){ + if (!msg_wrap->ParseFromArray(bin_data->data() + 6, msg_len)){ throw exc::EncodingException("Response is not well hexcoded"); } msg = msg_wrap; diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp index affd91553..9a43b3637 100644 --- a/src/device_trezor/trezor/transport.hpp +++ b/src/device_trezor/trezor/transport.hpp @@ -66,10 +66,12 @@ namespace trezor { // Base HTTP comm serialization. bool t_serialize(const std::string & in, std::string & out); + bool t_serialize(const epee::wipeable_string & in, std::string & out); bool t_serialize(const json_val & in, std::string & out); std::string t_serialize(const json_val & in); bool t_deserialize(const std::string & in, std::string & out); + bool t_deserialize(std::string & in, epee::wipeable_string & out); bool t_deserialize(const std::string & in, json & out); // Flexible json serialization. HTTP client tailored for bridge API @@ -84,6 +86,13 @@ namespace trezor { additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8")); const http::http_response_info* pri = nullptr; + const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { + if (!req_param.empty()) { + memwipe(&req_param[0], req_param.size()); + } + transport.wipe_response(); + }); + if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params))) { MERROR("Failed to invoke http request to " << uri); @@ -103,7 +112,7 @@ namespace trezor { return false; } - return t_deserialize(pri->m_body, result_struct); + return t_deserialize(const_cast<http::http_response_info*>(pri)->m_body, result_struct); } // Forward decl @@ -186,7 +195,7 @@ namespace trezor { std::string m_bridge_host; boost::optional<std::string> m_device_path; boost::optional<std::string> m_session; - boost::optional<std::string> m_response; + boost::optional<epee::wipeable_string> m_response; boost::optional<json> m_device_info; }; diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index 999894db0..70a4c1c8e 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -82,6 +82,7 @@ namespace cryptonote { rct::key sk = rct::scalarmultKey(rct::pk2rct(k), rct::sk2rct(blinded_skey)); crypto::secret_key msk = get_multisig_blinded_secret_key(rct::rct2sk(sk)); + memwipe(&sk, sizeof(sk)); multisig_keys.push_back(msk); sc_add(spend_skey.bytes, spend_skey.bytes, (const unsigned char*)msk.data); } @@ -126,10 +127,10 @@ namespace cryptonote //----------------------------------------------------------------- crypto::secret_key generate_multisig_view_secret_key(const crypto::secret_key &skey, const std::vector<crypto::secret_key> &skeys) { - rct::key view_skey = rct::sk2rct(get_multisig_blinded_secret_key(skey)); + crypto::secret_key view_skey = get_multisig_blinded_secret_key(skey); for (const auto &k: skeys) - sc_add(view_skey.bytes, view_skey.bytes, rct::sk2rct(k).bytes); - return rct::rct2sk(view_skey); + sc_add((unsigned char*)&view_skey, rct::sk2rct(view_skey).bytes, rct::sk2rct(k).bytes); + return view_skey; } //----------------------------------------------------------------- crypto::public_key generate_multisig_M_N_spend_public_key(const std::vector<crypto::public_key> &pkeys) diff --git a/src/net/zmq.cpp b/src/net/zmq.cpp index d02a22983..7ea80b907 100644 --- a/src/net/zmq.cpp +++ b/src/net/zmq.cpp @@ -33,6 +33,8 @@ #include <limits> #include <utility> +#include "byte_slice.h" + namespace net { namespace zmq @@ -183,6 +185,22 @@ namespace zmq { return retry_op(zmq_send, socket, payload.data(), payload.size(), flags); } + + expect<void> send(epee::byte_slice&& payload, void* socket, int flags) noexcept + { + void* const data = const_cast<std::uint8_t*>(payload.data()); + const std::size_t size = payload.size(); + auto buffer = payload.take_buffer(); // clears `payload` from callee + + zmq_msg_t msg{}; + MONERO_ZMQ_CHECK(zmq_msg_init_data(std::addressof(msg), data, size, epee::release_byte_slice::call, buffer.get())); + buffer.release(); // zmq will now decrement byte_slice ref-count + + expect<void> sent = retry_op(zmq_msg_send, std::addressof(msg), socket, flags); + if (!sent) // beware if removing `noexcept` from this function - possible leak here + zmq_msg_close(std::addressof(msg)); + return sent; + } } // zmq } // net diff --git a/src/net/zmq.h b/src/net/zmq.h index c6a7fd743..65560b62e 100644 --- a/src/net/zmq.h +++ b/src/net/zmq.h @@ -53,6 +53,11 @@ #define MONERO_ZMQ_THROW(msg) \ MONERO_THROW( ::net::zmq::get_error_code(), msg ) +namespace epee +{ + class byte_slice; +} + namespace net { namespace zmq @@ -132,5 +137,24 @@ namespace zmq \param flags See `zmq_send` for possible flags. \return `success()` if sent, otherwise ZMQ error. */ expect<void> send(epee::span<const std::uint8_t> payload, void* socket, int flags = 0) noexcept; + + /*! Sends `payload` on `socket`. Blocks until the entire message is queued + for sending, or until `zmq_term` is called on the `zmq_context` + associated with `socket`. If the context is terminated, + `make_error_code(ETERM)` is returned. + + \note This will automatically retry on `EINTR`, so exiting on + interrupts requires context termination. + \note If non-blocking behavior is requested on `socket` or by `flags`, + then `net::zmq::make_error_code(EAGAIN)` will be returned if this + would block. + + \param payload sent as one message on `socket`. + \param socket Handle created with `zmq_socket`. + \param flags See `zmq_msg_send` for possible flags. + + \post `payload.emtpy()` - ownership is transferred to zmq. + \return `success()` if sent, otherwise ZMQ error. */ + expect<void> send(epee::byte_slice&& payload, void* socket, int flags = 0) noexcept; } // zmq } // net diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 5337106dd..57b1335eb 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -384,6 +384,7 @@ namespace nodetool bool is_addr_recently_failed(const epee::net_utils::network_address& addr); bool is_priority_node(const epee::net_utils::network_address& na); std::set<std::string> get_seed_nodes(cryptonote::network_type nettype) const; + std::set<std::string> get_seed_nodes(); bool connect_to_seed(); template <class Container> @@ -467,7 +468,9 @@ namespace nodetool std::list<epee::net_utils::network_address> m_priority_peers; std::vector<epee::net_utils::network_address> m_exclusive_peers; std::vector<epee::net_utils::network_address> m_seed_nodes; - bool m_fallback_seed_nodes_added; + bool m_seed_nodes_initialized = false; + boost::shared_mutex m_seed_nodes_lock; + std::atomic_flag m_fallback_seed_nodes_added; std::vector<nodetool::peerlist_entry> m_command_line_peers; uint64_t m_peer_livetime; //keep connections to initiate some interactions diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 4c253b673..f8c177e3a 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -435,6 +435,8 @@ namespace nodetool if (command_line::has_arg(vm, arg_p2p_seed_node)) { + boost::unique_lock<boost::shared_mutex> lock(m_seed_nodes_lock); + if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes)) return false; } @@ -629,7 +631,120 @@ namespace nodetool full_addrs.insert("212.83.172.165:18080"); full_addrs.insert("192.110.160.146:18080"); full_addrs.insert("88.198.163.90:18080"); + full_addrs.insert("95.217.25.101:18080"); + full_addrs.insert("209.250.243.248:18080"); + full_addrs.insert("104.238.221.81:18080"); + full_addrs.insert("66.85.74.134:18080"); + } + return full_addrs; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + std::set<std::string> node_server<t_payload_net_handler>::get_seed_nodes() + { + if (!m_exclusive_peers.empty() || m_offline) + { + return {}; + } + if (m_nettype == cryptonote::TESTNET) + { + return get_seed_nodes(cryptonote::TESTNET); + } + if (m_nettype == cryptonote::STAGENET) + { + return get_seed_nodes(cryptonote::STAGENET); + } + + std::set<std::string> full_addrs; + + // for each hostname in the seed nodes list, attempt to DNS resolve and + // add the result addresses as seed nodes + // TODO: at some point add IPv6 support, but that won't be relevant + // for some time yet. + + std::vector<std::vector<std::string>> dns_results; + dns_results.resize(m_seed_nodes_list.size()); + + // some libc implementation provide only a very small stack + // for threads, e.g. musl only gives +- 80kb, which is not + // enough to do a resolve with unbound. we request a stack + // of 1 mb, which should be plenty + boost::thread::attributes thread_attributes; + thread_attributes.set_stack_size(1024*1024); + + std::list<boost::thread> dns_threads; + uint64_t result_index = 0; + for (const std::string& addr_str : m_seed_nodes_list) + { + boost::thread th = boost::thread(thread_attributes, [=, &dns_results, &addr_str] + { + MDEBUG("dns_threads[" << result_index << "] created for: " << addr_str); + // TODO: care about dnssec avail/valid + bool avail, valid; + std::vector<std::string> addr_list; + + try + { + addr_list = tools::DNSResolver::instance().get_ipv4(addr_str, avail, valid); + MDEBUG("dns_threads[" << result_index << "] DNS resolve done"); + boost::this_thread::interruption_point(); + } + catch(const boost::thread_interrupted&) + { + // thread interruption request + // even if we now have results, finish thread without setting + // result variables, which are now out of scope in main thread + MWARNING("dns_threads[" << result_index << "] interrupted"); + return; + } + + MINFO("dns_threads[" << result_index << "] addr_str: " << addr_str << " number of results: " << addr_list.size()); + dns_results[result_index] = addr_list; + }); + + dns_threads.push_back(std::move(th)); + ++result_index; + } + + MDEBUG("dns_threads created, now waiting for completion or timeout of " << CRYPTONOTE_DNS_TIMEOUT_MS << "ms"); + boost::chrono::system_clock::time_point deadline = boost::chrono::system_clock::now() + boost::chrono::milliseconds(CRYPTONOTE_DNS_TIMEOUT_MS); + uint64_t i = 0; + for (boost::thread& th : dns_threads) + { + if (! th.try_join_until(deadline)) + { + MWARNING("dns_threads[" << i << "] timed out, sending interrupt"); + th.interrupt(); + } + ++i; + } + + i = 0; + for (const auto& result : dns_results) + { + MDEBUG("DNS lookup for " << m_seed_nodes_list[i] << ": " << result.size() << " results"); + // if no results for node, thread's lookup likely timed out + if (result.size()) + { + for (const auto& addr_string : result) + full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT)); + } + ++i; + } + + // append the fallback nodes if we have too few seed nodes to start with + if (full_addrs.size() < MIN_WANTED_SEED_NODES) + { + if (full_addrs.empty()) + MINFO("DNS seed node lookup either timed out or failed, falling back to defaults"); + else + MINFO("Not enough DNS seed nodes found, using fallback defaults too"); + + for (const auto &peer: get_seed_nodes(cryptonote::MAINNET)) + full_addrs.insert(peer); + m_fallback_seed_nodes_added.test_and_set(); } + return full_addrs; } //----------------------------------------------------------------------------------- @@ -647,123 +762,21 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) { - std::set<std::string> full_addrs; - bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); - m_fallback_seed_nodes_added = false; if (m_nettype == cryptonote::TESTNET) { memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16); - full_addrs = get_seed_nodes(cryptonote::TESTNET); } else if (m_nettype == cryptonote::STAGENET) { memcpy(&m_network_id, &::config::stagenet::NETWORK_ID, 16); - full_addrs = get_seed_nodes(cryptonote::STAGENET); } else { memcpy(&m_network_id, &::config::NETWORK_ID, 16); - if (m_exclusive_peers.empty() && !m_offline) - { - // for each hostname in the seed nodes list, attempt to DNS resolve and - // add the result addresses as seed nodes - // TODO: at some point add IPv6 support, but that won't be relevant - // for some time yet. - - std::vector<std::vector<std::string>> dns_results; - dns_results.resize(m_seed_nodes_list.size()); - - // some libc implementation provide only a very small stack - // for threads, e.g. musl only gives +- 80kb, which is not - // enough to do a resolve with unbound. we request a stack - // of 1 mb, which should be plenty - boost::thread::attributes thread_attributes; - thread_attributes.set_stack_size(1024*1024); - - std::list<boost::thread> dns_threads; - uint64_t result_index = 0; - for (const std::string& addr_str : m_seed_nodes_list) - { - boost::thread th = boost::thread(thread_attributes, [=, &dns_results, &addr_str] - { - MDEBUG("dns_threads[" << result_index << "] created for: " << addr_str); - // TODO: care about dnssec avail/valid - bool avail, valid; - std::vector<std::string> addr_list; - - try - { - addr_list = tools::DNSResolver::instance().get_ipv4(addr_str, avail, valid); - MDEBUG("dns_threads[" << result_index << "] DNS resolve done"); - boost::this_thread::interruption_point(); - } - catch(const boost::thread_interrupted&) - { - // thread interruption request - // even if we now have results, finish thread without setting - // result variables, which are now out of scope in main thread - MWARNING("dns_threads[" << result_index << "] interrupted"); - return; - } - - MINFO("dns_threads[" << result_index << "] addr_str: " << addr_str << " number of results: " << addr_list.size()); - dns_results[result_index] = addr_list; - }); - - dns_threads.push_back(std::move(th)); - ++result_index; - } - - MDEBUG("dns_threads created, now waiting for completion or timeout of " << CRYPTONOTE_DNS_TIMEOUT_MS << "ms"); - boost::chrono::system_clock::time_point deadline = boost::chrono::system_clock::now() + boost::chrono::milliseconds(CRYPTONOTE_DNS_TIMEOUT_MS); - uint64_t i = 0; - for (boost::thread& th : dns_threads) - { - if (! th.try_join_until(deadline)) - { - MWARNING("dns_threads[" << i << "] timed out, sending interrupt"); - th.interrupt(); - } - ++i; - } - - i = 0; - for (const auto& result : dns_results) - { - MDEBUG("DNS lookup for " << m_seed_nodes_list[i] << ": " << result.size() << " results"); - // if no results for node, thread's lookup likely timed out - if (result.size()) - { - for (const auto& addr_string : result) - full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT)); - } - ++i; - } - - // append the fallback nodes if we have too few seed nodes to start with - if (full_addrs.size() < MIN_WANTED_SEED_NODES) - { - if (full_addrs.empty()) - MINFO("DNS seed node lookup either timed out or failed, falling back to defaults"); - else - MINFO("Not enough DNS seed nodes found, using fallback defaults too"); - - for (const auto &peer: get_seed_nodes(cryptonote::MAINNET)) - full_addrs.insert(peer); - m_fallback_seed_nodes_added = true; - } - } - } - - for (const auto& full_addr : full_addrs) - { - MDEBUG("Seed node: " << full_addr); - append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); } - MDEBUG("Number of seed nodes: " << m_seed_nodes.size()); m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir); network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); @@ -1540,6 +1553,20 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::connect_to_seed() { + boost::upgrade_lock<boost::shared_mutex> seed_nodes_upgrade_lock(m_seed_nodes_lock); + + if (!m_seed_nodes_initialized) + { + boost::upgrade_to_unique_lock<boost::shared_mutex> seed_nodes_lock(seed_nodes_upgrade_lock); + m_seed_nodes_initialized = true; + for (const auto& full_addr : get_seed_nodes()) + { + MDEBUG("Seed node: " << full_addr); + append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); + } + MDEBUG("Number of seed nodes: " << m_seed_nodes.size()); + } + if (m_seed_nodes.empty() || m_offline || !m_exclusive_peers.empty()) return true; @@ -1560,16 +1587,19 @@ namespace nodetool break; if(++try_count > m_seed_nodes.size()) { - if (!m_fallback_seed_nodes_added) + if (!m_fallback_seed_nodes_added.test_and_set()) { MWARNING("Failed to connect to any of seed peers, trying fallback seeds"); current_index = m_seed_nodes.size() - 1; - for (const auto &peer: get_seed_nodes(m_nettype)) { - MDEBUG("Fallback seed node: " << peer); - append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); + boost::upgrade_to_unique_lock<boost::shared_mutex> seed_nodes_lock(seed_nodes_upgrade_lock); + + for (const auto &peer: get_seed_nodes(m_nettype)) + { + MDEBUG("Fallback seed node: " << peer); + append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); + } } - m_fallback_seed_nodes_added = true; if (current_index == m_seed_nodes.size() - 1) { MWARNING("No fallback seeds, continuing without seeds"); @@ -1603,10 +1633,9 @@ namespace nodetool // Only have seeds in the public zone right now. size_t start_conn_count = get_public_outgoing_connections_count(); - if(!get_public_white_peers_count() && m_seed_nodes.size()) + if(!get_public_white_peers_count() && !connect_to_seed()) { - if (!connect_to_seed()) - return false; + return false; } if (!connect_to_peerlist(m_priority_peers)) return false; diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index 2ff88c6e7..6b88fd730 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -905,7 +905,7 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) rct::key m_y0 = rct::zero(), y1 = rct::zero(); int proof_data_index = 0; rct::keyV w_cache; - rct::keyV proof8_V, proof8_L, proof8_R; + std::vector<ge_p3> proof8_V, proof8_L, proof8_R; for (const Bulletproof *p: proofs) { const Bulletproof &proof = *p; @@ -918,13 +918,17 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) const rct::key weight_z = rct::skGen(); // pre-multiply some points by 8 - proof8_V.resize(proof.V.size()); for (size_t i = 0; i < proof.V.size(); ++i) proof8_V[i] = rct::scalarmult8(proof.V[i]); - proof8_L.resize(proof.L.size()); for (size_t i = 0; i < proof.L.size(); ++i) proof8_L[i] = rct::scalarmult8(proof.L[i]); - proof8_R.resize(proof.R.size()); for (size_t i = 0; i < proof.R.size(); ++i) proof8_R[i] = rct::scalarmult8(proof.R[i]); - rct::key proof8_T1 = rct::scalarmult8(proof.T1); - rct::key proof8_T2 = rct::scalarmult8(proof.T2); - rct::key proof8_S = rct::scalarmult8(proof.S); - rct::key proof8_A = rct::scalarmult8(proof.A); + proof8_V.resize(proof.V.size()); for (size_t i = 0; i < proof.V.size(); ++i) rct::scalarmult8(proof8_V[i], proof.V[i]); + proof8_L.resize(proof.L.size()); for (size_t i = 0; i < proof.L.size(); ++i) rct::scalarmult8(proof8_L[i], proof.L[i]); + proof8_R.resize(proof.R.size()); for (size_t i = 0; i < proof.R.size(); ++i) rct::scalarmult8(proof8_R[i], proof.R[i]); + ge_p3 proof8_T1; + ge_p3 proof8_T2; + ge_p3 proof8_S; + ge_p3 proof8_A; + rct::scalarmult8(proof8_T1, proof.T1); + rct::scalarmult8(proof8_T2, proof.T2); + rct::scalarmult8(proof8_S, proof.S); + rct::scalarmult8(proof8_A, proof.A); PERF_TIMER_START_BP(VERIFY_line_61); sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes); diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 6e4d063df..b2dd32ada 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -408,6 +408,18 @@ namespace rct { return res; } + //Computes 8P without byte conversion + void scalarmult8(ge_p3 &res, const key &P) + { + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + ge_p2 p2; + ge_p3_to_p2(&p2, &p3); + ge_p1p1 p1; + ge_mul8(&p1, &p2); + ge_p1p1_to_p3(&res, &p1); + } + //Computes lA where l is the curve order bool isInMainSubgroup(const key & A) { ge_p3 p3; diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index c24d48e9a..74e0ad833 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -124,6 +124,7 @@ namespace rct { key scalarmultH(const key & a); // multiplies a point by 8 key scalarmult8(const key & P); + void scalarmult8(ge_p3 &res, const key & P); // checks a is in the main subgroup (ie, not a small one) bool isInMainSubgroup(const key & a); diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index a7b265d63..2e3e7007e 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -29,6 +29,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "misc_log_ex.h" +#include "misc_language.h" #include "common/perf_timer.h" #include "common/threadpool.h" #include "common/util.h" @@ -108,6 +109,7 @@ namespace rct { //Borromean (c.f. gmax/andytoshi's paper) boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) { key64 L[2], alpha; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(alpha, sizeof(alpha));}); key c; int naught = 0, prime = 0, ii = 0, jj=0; boroSig bb; @@ -190,6 +192,7 @@ namespace rct { vector<geDsmp> Ip(dsRows); rv.II = keyV(dsRows); keyV alpha(rows); + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(alpha.data(), alpha.size() * sizeof(alpha[0]));}); keyV aG(rows); rv.ss = keyM(cols, aG); keyV aHP(dsRows); @@ -548,7 +551,7 @@ namespace rct { subKeys(M[i][1], pubs[i].mask, Cout); } mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); - memwipe(&sk[0], sizeof(key)); + memwipe(sk.data(), sk.size() * sizeof(key)); return result; } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index bf4b7b4aa..9b7f26a02 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -48,6 +48,7 @@ extern "C" { #include "hex.h" #include "span.h" +#include "memwipe.h" #include "serialization/vector.h" #include "serialization/debug_archive.h" #include "serialization/binary_archive.h" @@ -106,6 +107,8 @@ namespace rct { key L; key R; key ki; + + ~multisig_kLRki() { memwipe(&k, sizeof(k)); } }; struct multisig_out { diff --git a/src/rpc/bootstrap_node_selector.h b/src/rpc/bootstrap_node_selector.h index fc993719b..47722a008 100644 --- a/src/rpc/bootstrap_node_selector.h +++ b/src/rpc/bootstrap_node_selector.h @@ -54,6 +54,8 @@ namespace bootstrap_node struct selector { + virtual ~selector() = default; + virtual void handle_result(const std::string &address, bool success) = 0; virtual boost::optional<node_info> next_node() = 0; }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 1fd0c037b..a789a7a1a 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -224,7 +224,7 @@ namespace cryptonote } else if (address == "auto") { - auto get_nodes = [this, credits_per_hash_threshold]() { + auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled)); @@ -438,7 +438,7 @@ namespace cryptonote store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase - res.tx_pool_size = m_core.get_pool_transactions_count(); + res.tx_pool_size = m_core.get_pool_transactions_count(!restricted); res.alt_blocks_count = restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); uint64_t total_conn = restricted ? 0 : m_p2p.get_public_connections_count(); res.outgoing_connections_count = restricted ? 0 : m_p2p.get_public_outgoing_connections_count(); @@ -1119,6 +1119,8 @@ namespace cryptonote } res.sanity_check_failed = false; + const bool restricted = m_restricted && ctx; + tx_verification_context tvc{}; if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, (req.do_not_relay ? relay_method::none : relay_method::local), false) || tvc.m_verifivation_failed) { @@ -1423,12 +1425,13 @@ namespace cryptonote const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; + const bool allow_sensitive = !request_has_rpc_origin || !restricted; - size_t n_txes = m_core.get_pool_transactions_count(); + size_t n_txes = m_core.get_pool_transactions_count(allow_sensitive); if (n_txes > 0) { CHECK_PAYMENT_SAME_TS(req, res, n_txes * COST_PER_TX); - m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !restricted); + m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, allow_sensitive); for (tx_info& txi : res.transactions) txi.tx_blob = epee::string_tools::buff_to_hex_nodelimer(txi.tx_blob); } @@ -1448,12 +1451,13 @@ namespace cryptonote const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; + const bool allow_sensitive = !request_has_rpc_origin || !restricted; - size_t n_txes = m_core.get_pool_transactions_count(); + size_t n_txes = m_core.get_pool_transactions_count(allow_sensitive); if (n_txes > 0) { CHECK_PAYMENT_SAME_TS(req, res, n_txes * COST_PER_POOL_HASH); - m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !restricted); + m_core.get_pool_transaction_hashes(res.tx_hashes, allow_sensitive); } res.status = CORE_RPC_STATUS_OK; @@ -1471,13 +1475,14 @@ namespace cryptonote const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; + const bool allow_sensitive = !request_has_rpc_origin || !restricted; - size_t n_txes = m_core.get_pool_transactions_count(); + size_t n_txes = m_core.get_pool_transactions_count(allow_sensitive); if (n_txes > 0) { CHECK_PAYMENT_SAME_TS(req, res, n_txes * COST_PER_POOL_HASH); std::vector<crypto::hash> tx_hashes; - m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !restricted); + m_core.get_pool_transaction_hashes(tx_hashes, allow_sensitive); res.tx_hashes.reserve(tx_hashes.size()); for (const crypto::hash &tx_hash: tx_hashes) res.tx_hashes.push_back(epee::string_tools::pod_to_hex(tx_hash)); diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index d05854e34..de0510fec 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -33,6 +33,7 @@ #include <stdexcept> #include <boost/uuid/nil_generator.hpp> +#include <boost/utility/string_ref.hpp> // likely included by daemon_handler.h's includes, // but including here for clarity #include "cryptonote_core/cryptonote_core.h" @@ -48,7 +49,7 @@ namespace rpc { namespace { - using handler_function = std::string(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& msg); + using handler_function = epee::byte_slice(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& msg); struct handler_map { const char* method_name; @@ -66,7 +67,7 @@ namespace rpc } template<typename Message> - std::string handle_message(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& parameters) + epee::byte_slice handle_message(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& parameters) { typename Message::Request request{}; request.fromJson(parameters); @@ -903,7 +904,7 @@ namespace rpc return true; } - std::string DaemonHandler::handle(const std::string& request) + epee::byte_slice DaemonHandler::handle(const std::string& request) { MDEBUG("Handling RPC request: " << request); @@ -916,8 +917,11 @@ namespace rpc if (matched_handler == std::end(handlers) || matched_handler->method_name != request_type) return BAD_REQUEST(request_type, req_full.getID()); - std::string response = matched_handler->call(*this, req_full.getID(), req_full.getMessage()); - MDEBUG("Returning RPC response: " << response); + epee::byte_slice response = matched_handler->call(*this, req_full.getID(), req_full.getMessage()); + + const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; + MDEBUG("Returning RPC response: " << response_view); + return response; } catch (const std::exception& e) diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 61eac17f0..b797b1155 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -28,6 +28,7 @@ #pragma once +#include "byte_slice.h" #include "daemon_messages.h" #include "daemon_rpc_version.h" #include "rpc_handler.h" @@ -132,7 +133,7 @@ class DaemonHandler : public RpcHandler void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); - std::string handle(const std::string& request); + epee::byte_slice handle(const std::string& request) override final; private: diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index 5c179408e..22f73472d 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -34,14 +34,14 @@ namespace cryptonote namespace rpc { -void GetHeight::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetHeight::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void GetHeight::Request::fromJson(const rapidjson::Value& val) { } -void GetHeight::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, height, height); } @@ -52,7 +52,7 @@ void GetHeight::Response::fromJson(const rapidjson::Value& val) } -void GetBlocksFast::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlocksFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, block_ids, block_ids); INSERT_INTO_JSON_OBJECT(dest, start_height, start_height); @@ -66,7 +66,7 @@ void GetBlocksFast::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, prune, prune); } -void GetBlocksFast::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlocksFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, blocks, blocks); INSERT_INTO_JSON_OBJECT(dest, start_height, start_height); @@ -83,7 +83,7 @@ void GetBlocksFast::Response::fromJson(const rapidjson::Value& val) } -void GetHashesFast::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetHashesFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, known_hashes, known_hashes); INSERT_INTO_JSON_OBJECT(dest, start_height, start_height); @@ -95,7 +95,7 @@ void GetHashesFast::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, start_height, start_height); } -void GetHashesFast::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetHashesFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, hashes, hashes); INSERT_INTO_JSON_OBJECT(dest, start_height, start_height); @@ -110,7 +110,7 @@ void GetHashesFast::Response::fromJson(const rapidjson::Value& val) } -void GetTransactions::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetTransactions::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, tx_hashes, tx_hashes); } @@ -120,7 +120,7 @@ void GetTransactions::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, tx_hashes, tx_hashes); } -void GetTransactions::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetTransactions::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, txs, txs); INSERT_INTO_JSON_OBJECT(dest, missed_hashes, missed_hashes); @@ -133,7 +133,7 @@ void GetTransactions::Response::fromJson(const rapidjson::Value& val) } -void KeyImagesSpent::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void KeyImagesSpent::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, key_images, key_images); } @@ -143,7 +143,7 @@ void KeyImagesSpent::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, key_images, key_images); } -void KeyImagesSpent::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void KeyImagesSpent::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, spent_status, spent_status); } @@ -154,7 +154,7 @@ void KeyImagesSpent::Response::fromJson(const rapidjson::Value& val) } -void GetTxGlobalOutputIndices::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetTxGlobalOutputIndices::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, tx_hash, tx_hash); } @@ -164,7 +164,7 @@ void GetTxGlobalOutputIndices::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, tx_hash, tx_hash); } -void GetTxGlobalOutputIndices::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetTxGlobalOutputIndices::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, output_indices, output_indices); } @@ -174,7 +174,7 @@ void GetTxGlobalOutputIndices::Response::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } -void SendRawTx::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SendRawTx::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, tx, tx); INSERT_INTO_JSON_OBJECT(dest, relay, relay); @@ -186,7 +186,7 @@ void SendRawTx::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, relay, relay); } -void SendRawTx::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SendRawTx::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, relayed, relayed); } @@ -197,7 +197,7 @@ void SendRawTx::Response::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, relayed, relayed); } -void SendRawTxHex::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SendRawTxHex::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, tx_as_hex, tx_as_hex); INSERT_INTO_JSON_OBJECT(dest, relay, relay); @@ -209,7 +209,7 @@ void SendRawTxHex::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, relay, relay); } -void StartMining::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void StartMining::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, miner_address, miner_address); INSERT_INTO_JSON_OBJECT(dest, threads_count, threads_count); @@ -225,7 +225,7 @@ void StartMining::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, ignore_battery, ignore_battery); } -void StartMining::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void StartMining::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void StartMining::Response::fromJson(const rapidjson::Value& val) @@ -233,14 +233,14 @@ void StartMining::Response::fromJson(const rapidjson::Value& val) } -void StopMining::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void StopMining::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void StopMining::Request::fromJson(const rapidjson::Value& val) { } -void StopMining::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void StopMining::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void StopMining::Response::fromJson(const rapidjson::Value& val) @@ -248,14 +248,14 @@ void StopMining::Response::fromJson(const rapidjson::Value& val) } -void MiningStatus::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void MiningStatus::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void MiningStatus::Request::fromJson(const rapidjson::Value& val) { } -void MiningStatus::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void MiningStatus::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, active, active); INSERT_INTO_JSON_OBJECT(dest, speed, speed); @@ -274,14 +274,14 @@ void MiningStatus::Response::fromJson(const rapidjson::Value& val) } -void GetInfo::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetInfo::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void GetInfo::Request::fromJson(const rapidjson::Value& val) { } -void GetInfo::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, info, info); } @@ -292,14 +292,14 @@ void GetInfo::Response::fromJson(const rapidjson::Value& val) } -void SaveBC::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SaveBC::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void SaveBC::Request::fromJson(const rapidjson::Value& val) { } -void SaveBC::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SaveBC::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void SaveBC::Response::fromJson(const rapidjson::Value& val) @@ -307,7 +307,7 @@ void SaveBC::Response::fromJson(const rapidjson::Value& val) } -void GetBlockHash::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHash::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, height, height); } @@ -317,7 +317,7 @@ void GetBlockHash::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, height, height); } -void GetBlockHash::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHash::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, hash, hash); } @@ -328,14 +328,14 @@ void GetBlockHash::Response::fromJson(const rapidjson::Value& val) } -void GetLastBlockHeader::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetLastBlockHeader::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void GetLastBlockHeader::Request::fromJson(const rapidjson::Value& val) { } -void GetLastBlockHeader::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetLastBlockHeader::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, header, header); } @@ -346,7 +346,7 @@ void GetLastBlockHeader::Response::fromJson(const rapidjson::Value& val) } -void GetBlockHeaderByHash::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHeaderByHash::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, hash, hash); } @@ -356,7 +356,7 @@ void GetBlockHeaderByHash::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, hash, hash); } -void GetBlockHeaderByHash::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHeaderByHash::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, header, header); } @@ -367,7 +367,7 @@ void GetBlockHeaderByHash::Response::fromJson(const rapidjson::Value& val) } -void GetBlockHeaderByHeight::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHeaderByHeight::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, height, height); } @@ -377,7 +377,7 @@ void GetBlockHeaderByHeight::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, height, height); } -void GetBlockHeaderByHeight::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHeaderByHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, header, header); } @@ -388,7 +388,7 @@ void GetBlockHeaderByHeight::Response::fromJson(const rapidjson::Value& val) } -void GetBlockHeadersByHeight::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHeadersByHeight::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, heights, heights); } @@ -398,7 +398,7 @@ void GetBlockHeadersByHeight::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, heights, heights); } -void GetBlockHeadersByHeight::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetBlockHeadersByHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, headers, headers); } @@ -409,14 +409,14 @@ void GetBlockHeadersByHeight::Response::fromJson(const rapidjson::Value& val) } -void GetPeerList::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetPeerList::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void GetPeerList::Request::fromJson(const rapidjson::Value& val) { } -void GetPeerList::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetPeerList::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, white_list, white_list); INSERT_INTO_JSON_OBJECT(dest, gray_list, gray_list); @@ -429,7 +429,7 @@ void GetPeerList::Response::fromJson(const rapidjson::Value& val) } -void SetLogLevel::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SetLogLevel::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, level, level); } @@ -439,7 +439,7 @@ void SetLogLevel::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, level, level); } -void SetLogLevel::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void SetLogLevel::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void SetLogLevel::Response::fromJson(const rapidjson::Value& val) @@ -447,14 +447,14 @@ void SetLogLevel::Response::fromJson(const rapidjson::Value& val) } -void GetTransactionPool::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetTransactionPool::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void GetTransactionPool::Request::fromJson(const rapidjson::Value& val) { } -void GetTransactionPool::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetTransactionPool::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, transactions, transactions); INSERT_INTO_JSON_OBJECT(dest, key_images, key_images); @@ -467,7 +467,7 @@ void GetTransactionPool::Response::fromJson(const rapidjson::Value& val) } -void HardForkInfo::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void HardForkInfo::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, version, version); } @@ -477,7 +477,7 @@ void HardForkInfo::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, version, version); } -void HardForkInfo::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void HardForkInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, info, info); } @@ -488,7 +488,7 @@ void HardForkInfo::Response::fromJson(const rapidjson::Value& val) } -void GetOutputHistogram::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetOutputHistogram::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, amounts, amounts); INSERT_INTO_JSON_OBJECT(dest, min_count, min_count); @@ -506,7 +506,7 @@ void GetOutputHistogram::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, recent_cutoff, recent_cutoff); } -void GetOutputHistogram::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetOutputHistogram::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, histogram, histogram); } @@ -517,7 +517,7 @@ void GetOutputHistogram::Response::fromJson(const rapidjson::Value& val) } -void GetOutputKeys::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetOutputKeys::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, outputs, outputs); } @@ -527,7 +527,7 @@ void GetOutputKeys::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, outputs, outputs); } -void GetOutputKeys::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetOutputKeys::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, keys, keys); } @@ -538,14 +538,14 @@ void GetOutputKeys::Response::fromJson(const rapidjson::Value& val) } -void GetRPCVersion::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetRPCVersion::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} void GetRPCVersion::Request::fromJson(const rapidjson::Value& val) { } -void GetRPCVersion::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetRPCVersion::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, version, version); } @@ -555,7 +555,7 @@ void GetRPCVersion::Response::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, version, version); } -void GetFeeEstimate::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetFeeEstimate::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, num_grace_blocks, num_grace_blocks); } @@ -565,7 +565,7 @@ void GetFeeEstimate::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks); } -void GetFeeEstimate::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetFeeEstimate::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, estimated_base_fee, estimated_base_fee); INSERT_INTO_JSON_OBJECT(dest, fee_mask, fee_mask); @@ -581,7 +581,7 @@ void GetFeeEstimate::Response::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version); } -void GetOutputDistribution::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetOutputDistribution::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, amounts, amounts); INSERT_INTO_JSON_OBJECT(dest, from_height, from_height); @@ -597,7 +597,7 @@ void GetOutputDistribution::Request::fromJson(const rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, cumulative, cumulative); } -void GetOutputDistribution::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void GetOutputDistribution::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const { INSERT_INTO_JSON_OBJECT(dest, status, status); INSERT_INTO_JSON_OBJECT(dest, distributions, distributions); diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index bb5059cdc..64ea3e9d4 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -28,11 +28,11 @@ #pragma once -#include <rapidjson/stringbuffer.h> #include <rapidjson/writer.h> #include <unordered_map> #include <vector> +#include "byte_stream.h" #include "message.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "rpc/message_data_structs.h" @@ -50,7 +50,7 @@ class classname \ public: \ Request() { } \ ~Request() { } \ - void doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const override final; \ + void doToJson(rapidjson::Writer<epee::byte_stream>& dest) const override final; \ void fromJson(const rapidjson::Value& val) override final; #define BEGIN_RPC_MESSAGE_RESPONSE \ @@ -59,7 +59,7 @@ class classname \ public: \ Response() { } \ ~Response() { } \ - void doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const override final; \ + void doToJson(rapidjson::Writer<epee::byte_stream>& dest) const override final; \ void fromJson(const rapidjson::Value& val) override final; #define END_RPC_MESSAGE_REQUEST }; diff --git a/src/rpc/message.cpp b/src/rpc/message.cpp index 5b6a1c05b..fffb44921 100644 --- a/src/rpc/message.cpp +++ b/src/rpc/message.cpp @@ -62,7 +62,7 @@ const rapidjson::Value& get_method_field(const rapidjson::Value& src) } } -void Message::toJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const +void Message::toJson(rapidjson::Writer<epee::byte_stream>& dest) const { dest.StartObject(); INSERT_INTO_JSON_OBJECT(dest, status, status); @@ -149,11 +149,11 @@ cryptonote::rpc::error FullMessage::getError() return err; } -std::string FullMessage::getRequest(const std::string& request, const Message& message, const unsigned id) +epee::byte_slice FullMessage::getRequest(const std::string& request, const Message& message, const unsigned id) { - rapidjson::StringBuffer buffer; + epee::byte_stream buffer; { - rapidjson::Writer<rapidjson::StringBuffer> dest{buffer}; + rapidjson::Writer<epee::byte_stream> dest{buffer}; dest.StartObject(); INSERT_INTO_JSON_OBJECT(dest, jsonrpc, (boost::string_ref{"2.0", 3})); @@ -172,15 +172,15 @@ std::string FullMessage::getRequest(const std::string& request, const Message& m if (!dest.IsComplete()) throw std::logic_error{"Invalid JSON tree generated"}; } - return std::string{buffer.GetString(), buffer.GetSize()}; + return epee::byte_slice{std::move(buffer)}; } -std::string FullMessage::getResponse(const Message& message, const rapidjson::Value& id) +epee::byte_slice FullMessage::getResponse(const Message& message, const rapidjson::Value& id) { - rapidjson::StringBuffer buffer; + epee::byte_stream buffer; { - rapidjson::Writer<rapidjson::StringBuffer> dest{buffer}; + rapidjson::Writer<epee::byte_stream> dest{buffer}; dest.StartObject(); INSERT_INTO_JSON_OBJECT(dest, jsonrpc, (boost::string_ref{"2.0", 3})); @@ -207,17 +207,17 @@ std::string FullMessage::getResponse(const Message& message, const rapidjson::Va if (!dest.IsComplete()) throw std::logic_error{"Invalid JSON tree generated"}; } - return std::string{buffer.GetString(), buffer.GetSize()}; + return epee::byte_slice{std::move(buffer)}; } // convenience functions for bad input -std::string BAD_REQUEST(const std::string& request) +epee::byte_slice BAD_REQUEST(const std::string& request) { rapidjson::Value invalid; return BAD_REQUEST(request, invalid); } -std::string BAD_REQUEST(const std::string& request, const rapidjson::Value& id) +epee::byte_slice BAD_REQUEST(const std::string& request, const rapidjson::Value& id) { Message fail; fail.status = Message::STATUS_BAD_REQUEST; @@ -225,7 +225,7 @@ std::string BAD_REQUEST(const std::string& request, const rapidjson::Value& id) return FullMessage::getResponse(fail, id); } -std::string BAD_JSON(const std::string& error_details) +epee::byte_slice BAD_JSON(const std::string& error_details) { rapidjson::Value invalid; Message fail; diff --git a/src/rpc/message.h b/src/rpc/message.h index 4cbc84fe4..5c369cdfc 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -29,10 +29,11 @@ #pragma once #include <rapidjson/document.h> -#include <rapidjson/stringbuffer.h> #include <rapidjson/writer.h> #include <string> +#include "byte_slice.h" +#include "byte_stream.h" #include "rpc/message_data_structs.h" namespace cryptonote @@ -43,7 +44,7 @@ namespace rpc class Message { - virtual void doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const + virtual void doToJson(rapidjson::Writer<epee::byte_stream>& dest) const {} public: @@ -57,7 +58,7 @@ namespace rpc virtual ~Message() { } - void toJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const; + void toJson(rapidjson::Writer<epee::byte_stream>& dest) const; virtual void fromJson(const rapidjson::Value& val); @@ -85,8 +86,8 @@ namespace rpc cryptonote::rpc::error getError(); - static std::string getRequest(const std::string& request, const Message& message, unsigned id); - static std::string getResponse(const Message& message, const rapidjson::Value& id); + static epee::byte_slice getRequest(const std::string& request, const Message& message, unsigned id); + static epee::byte_slice getResponse(const Message& message, const rapidjson::Value& id); private: FullMessage() = default; @@ -99,10 +100,10 @@ namespace rpc // convenience functions for bad input - std::string BAD_REQUEST(const std::string& request); - std::string BAD_REQUEST(const std::string& request, const rapidjson::Value& id); + epee::byte_slice BAD_REQUEST(const std::string& request); + epee::byte_slice BAD_REQUEST(const std::string& request, const rapidjson::Value& id); - std::string BAD_JSON(const std::string& error_details); + epee::byte_slice BAD_JSON(const std::string& error_details); } // namespace rpc diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index b81983d28..9a1c3fc12 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -32,6 +32,7 @@ #include <cstdint> #include <string> #include <vector> +#include "byte_slice.h" #include "crypto/hash.h" namespace cryptonote @@ -54,7 +55,7 @@ class RpcHandler RpcHandler() { } virtual ~RpcHandler() { } - virtual std::string handle(const std::string& request) = 0; + virtual epee::byte_slice handle(const std::string& request) = 0; static boost::optional<output_distribution_data> get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height); diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 1ee55673e..0d595539d 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -32,6 +32,8 @@ #include <cstdint> #include <system_error> +#include "byte_slice.h" + namespace cryptonote { @@ -73,10 +75,11 @@ void ZmqServer::serve() { const std::string message = MONERO_UNWRAP(net::zmq::receive(socket.get())); MDEBUG("Received RPC request: \"" << message << "\""); - const std::string& response = handler.handle(message); + epee::byte_slice response = handler.handle(message); - MONERO_UNWRAP(net::zmq::send(epee::strspan<std::uint8_t>(response), socket.get())); - MDEBUG("Sent RPC reply: \"" << response << "\""); + const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; + MDEBUG("Sending RPC reply: \"" << response_view << "\""); + MONERO_UNWRAP(net::zmq::send(std::move(response), socket.get())); } } catch (const std::system_error& e) diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index f20fd181a..6228b4bec 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -126,12 +126,12 @@ void read_hex(const rapidjson::Value& val, epee::span<std::uint8_t> dest) } } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rapidjson::Value& src) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rapidjson::Value& src) { src.Accept(dest); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const boost::string_ref i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const boost::string_ref i) { dest.String(i.data(), i.size()); } @@ -146,7 +146,7 @@ void fromJsonValue(const rapidjson::Value& val, std::string& str) str = val.GetString(); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, bool i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i) { dest.Bool(i); } @@ -185,7 +185,7 @@ void fromJsonValue(const rapidjson::Value& val, short& i) to_int(val, i); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned int i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned int i) { dest.Uint(i); } @@ -195,7 +195,7 @@ void fromJsonValue(const rapidjson::Value& val, unsigned int& i) to_uint(val, i); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const int i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const int i) { dest.Int(i); } @@ -205,7 +205,7 @@ void fromJsonValue(const rapidjson::Value& val, int& i) to_int(val, i); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long long i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long long i) { static_assert(!precision_loss<unsigned long long, std::uint64_t>(), "bad uint64 conversion"); dest.Uint64(i); @@ -216,7 +216,7 @@ void fromJsonValue(const rapidjson::Value& val, unsigned long long& i) to_uint64(val, i); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long long i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long long i) { static_assert(!precision_loss<long long, std::int64_t>(), "bad int64 conversion"); dest.Int64(i); @@ -237,7 +237,7 @@ void fromJsonValue(const rapidjson::Value& val, long& i) to_int64(val, i); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::transaction& tx) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::transaction& tx) { dest.StartObject(); @@ -269,7 +269,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx) GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block& b) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b) { dest.StartObject(); @@ -301,14 +301,14 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block& b) GET_FROM_JSON_OBJECT(val, b.tx_hashes, tx_hashes); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_v& txin) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_v& txin) { dest.StartObject(); struct add_input { using result_type = void; - rapidjson::Writer<rapidjson::StringBuffer>& dest; + rapidjson::Writer<epee::byte_stream>& dest; void operator()(cryptonote::txin_to_key const& input) const { @@ -373,7 +373,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin) } } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_gen& txin) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_gen& txin) { dest.StartObject(); @@ -392,7 +392,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_gen& txin) GET_FROM_JSON_OBJECT(val, txin.height, height); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_script& txin) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_script& txin) { dest.StartObject(); @@ -417,7 +417,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_script& txin } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_scripthash& txin) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_scripthash& txin) { dest.StartObject(); @@ -443,7 +443,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_scripthash& GET_FROM_JSON_OBJECT(val, txin.sigset, sigset); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_key& txin) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_key& txin) { dest.StartObject(); @@ -467,7 +467,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin) } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_script& txout) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_script& txout) { dest.StartObject(); @@ -489,7 +489,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_script& txo } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_scripthash& txout) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_scripthash& txout) { dest.StartObject(); @@ -509,7 +509,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_scripthash& } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_key& txout) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_key& txout) { dest.StartObject(); @@ -528,7 +528,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_key& txout) GET_FROM_JSON_OBJECT(val, txout.key, key); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_out& txout) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_out& txout) { dest.StartObject(); INSERT_INTO_JSON_OBJECT(dest, amount, txout.amount); @@ -537,7 +537,7 @@ void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const crypton { using result_type = void; - rapidjson::Writer<rapidjson::StringBuffer>& dest; + rapidjson::Writer<epee::byte_stream>& dest; void operator()(cryptonote::txout_to_key const& output) const { @@ -596,7 +596,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) } } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::connection_info& info) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::connection_info& info) { dest.StartObject(); @@ -668,7 +668,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& inf GET_FROM_JSON_OBJECT(val, info.current_upload, current_upload); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_blob_entry& tx) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_blob_entry& tx) { dest.StartObject(); @@ -689,7 +689,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx) GET_FROM_JSON_OBJECT(val, tx.prunable_hash, prunable_hash); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block_complete_entry& blk) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block_complete_entry& blk) { dest.StartObject(); @@ -711,7 +711,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry GET_FROM_JSON_OBJECT(val, blk.txs, transactions); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::block_with_transactions& blk) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::block_with_transactions& blk) { dest.StartObject(); @@ -733,7 +733,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::block_with_tran GET_FROM_JSON_OBJECT(val, blk.transactions, transactions); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::transaction_info& tx_info) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::transaction_info& tx_info) { dest.StartObject(); @@ -757,7 +757,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::transaction_inf GET_FROM_JSON_OBJECT(val, tx_info.transaction, transaction); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_and_amount_index& out) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_and_amount_index& out) { dest.StartObject(); @@ -779,7 +779,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_and_ GET_FROM_JSON_OBJECT(val, out.key, key); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::amount_with_random_outputs& out) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::amount_with_random_outputs& out) { dest.StartObject(); @@ -801,7 +801,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::amount_with_ran GET_FROM_JSON_OBJECT(val, out.outputs, outputs); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::peer& peer) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::peer& peer) { dest.StartObject(); @@ -833,7 +833,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer) GET_FROM_JSON_OBJECT(val, peer.pruning_seed, pruning_seed); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::tx_in_pool& tx) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::tx_in_pool& tx) { dest.StartObject(); @@ -880,7 +880,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx) GET_FROM_JSON_OBJECT(val, tx.double_spend_seen, double_spend_seen); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::hard_fork_info& info) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::hard_fork_info& info) { dest.StartObject(); @@ -914,7 +914,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::hard_fork_info& GET_FROM_JSON_OBJECT(val, info.earliest_height, earliest_height); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_count& out) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_count& out) { dest.StartObject(); @@ -940,7 +940,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_c GET_FROM_JSON_OBJECT(val, out.recent_count, recent_count); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_and_index& out) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_and_index& out) { dest.StartObject(); @@ -962,7 +962,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_a GET_FROM_JSON_OBJECT(val, out.index, index); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_mask_unlocked& out) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_mask_unlocked& out) { dest.StartObject(); @@ -985,7 +985,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_mask GET_FROM_JSON_OBJECT(val, out.unlocked, unlocked); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::error& err) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::error& err) { dest.StartObject(); @@ -1008,7 +1008,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::error& error) GET_FROM_JSON_OBJECT(val, error.message, message); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::BlockHeaderResponse& response) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::BlockHeaderResponse& response) { dest.StartObject(); @@ -1045,7 +1045,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp GET_FROM_JSON_OBJECT(val, response.reward, reward); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rctSig& sig) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& sig) { using boost::adaptors::transform; @@ -1115,7 +1115,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig) } } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::ecdhTuple& tuple) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple) { dest.StartObject(); INSERT_INTO_JSON_OBJECT(dest, mask, tuple.mask); @@ -1134,7 +1134,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple) GET_FROM_JSON_OBJECT(val, tuple.amount, amount); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rangeSig& sig) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rangeSig& sig) { dest.StartObject(); @@ -1171,7 +1171,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig) } } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::Bulletproof& p) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::Bulletproof& p) { dest.StartObject(); @@ -1212,7 +1212,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::Bulletproof& p) GET_FROM_JSON_OBJECT(val, p.t, t); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::boroSig& sig) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::boroSig& sig) { dest.StartObject(); @@ -1257,7 +1257,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig) GET_FROM_JSON_OBJECT(val, sig.ee, ee); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::mgSig& sig) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig) { dest.StartObject(); @@ -1278,7 +1278,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig) GET_FROM_JSON_OBJECT(val, sig.cc, cc); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::DaemonInfo& info) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info) { dest.StartObject(); @@ -1339,7 +1339,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_distribution& dist) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist) { dest.StartObject(); diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 664b539b5..2a9b63b08 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -31,9 +31,9 @@ #include <boost/utility/string_ref.hpp> #include <cstring> #include <rapidjson/document.h> -#include <rapidjson/stringbuffer.h> #include <rapidjson/writer.h> +#include "byte_stream.h" #include "cryptonote_basic/cryptonote_basic.h" #include "rpc/message_data_structs.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" @@ -123,7 +123,7 @@ void read_hex(const rapidjson::Value& val, epee::span<std::uint8_t> dest); // POD to json key template <class Type> -inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Type& pod) +inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Writer<epee::byte_stream>& dest, const Type& pod) { const auto hex = epee::to_hex::array(pod); dest.Key(hex.data(), hex.size()); @@ -131,7 +131,7 @@ inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Wri // POD to json value template <class Type> -inline typename std::enable_if<is_to_hex<Type>()>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Type& pod) +inline typename std::enable_if<is_to_hex<Type>()>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Type& pod) { const auto hex = epee::to_hex::array(pod); dest.String(hex.data(), hex.size()); @@ -144,16 +144,16 @@ inline typename std::enable_if<is_to_hex<Type>()>::type fromJsonValue(const rapi json::read_hex(val, epee::as_mut_byte_span(t)); } -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rapidjson::Value& src); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rapidjson::Value& src); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, boost::string_ref i); -inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const std::string& i) +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, boost::string_ref i); +inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const std::string& i) { toJsonValue(dest, boost::string_ref{i}); } void fromJsonValue(const rapidjson::Value& val, std::string& str); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, bool i); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i); void fromJsonValue(const rapidjson::Value& val, bool& b); // integers overloads for toJsonValue are not needed for standard promotions @@ -168,144 +168,144 @@ void fromJsonValue(const rapidjson::Value& val, unsigned short& i); void fromJsonValue(const rapidjson::Value& val, short& i); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned i); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned i); void fromJsonValue(const rapidjson::Value& val, unsigned& i); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const int); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const int); void fromJsonValue(const rapidjson::Value& val, int& i); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long long i); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long long i); void fromJsonValue(const rapidjson::Value& val, unsigned long long& i); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long long i); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long long i); void fromJsonValue(const rapidjson::Value& val, long long& i); -inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long i) { +inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long i) { toJsonValue(dest, static_cast<unsigned long long>(i)); } void fromJsonValue(const rapidjson::Value& val, unsigned long& i); -inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long i) { +inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long i) { toJsonValue(dest, static_cast<long long>(i)); } void fromJsonValue(const rapidjson::Value& val, long& i); // end integers -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::transaction& tx); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::transaction& tx); void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block& b); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b); void fromJsonValue(const rapidjson::Value& val, cryptonote::block& b); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_v& txin); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_v& txin); void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_gen& txin); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_gen& txin); void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_gen& txin); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_script& txin); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_script& txin); void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_script& txin); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_scripthash& txin); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_scripthash& txin); void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_scripthash& txin); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_key& txin); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_key& txin); void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_target_v& txout); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_target_v& txout); void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_target_v& txout); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_script& txout); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_script& txout); void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_script& txout); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_scripthash& txout); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_scripthash& txout); void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_scripthash& txout); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_key& txout); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_key& txout); void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_key& txout); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_out& txout); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_out& txout); void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::connection_info& info); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::connection_info& info); void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& info); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_blob_entry& tx); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_blob_entry& tx); void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block_complete_entry& blk); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block_complete_entry& blk); void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry& blk); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::block_with_transactions& blk); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::block_with_transactions& blk); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::block_with_transactions& blk); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::transaction_info& tx_info); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::transaction_info& tx_info); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::transaction_info& tx_info); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_and_amount_index& out); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_and_amount_index& out); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_and_amount_index& out); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::amount_with_random_outputs& out); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::amount_with_random_outputs& out); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::amount_with_random_outputs& out); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::peer& peer); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::peer& peer); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::tx_in_pool& tx); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::tx_in_pool& tx); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::hard_fork_info& info); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::hard_fork_info& info); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::hard_fork_info& info); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_count& out); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_count& out); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_count& out); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_and_index& out); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_and_index& out); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_and_index& out); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_mask_unlocked& out); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_mask_unlocked& out); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_mask_unlocked& out); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::error& err); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::error& err); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::error& error); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::BlockHeaderResponse& response); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::BlockHeaderResponse& response); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResponse& response); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rctSig& i); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i); void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::ecdhTuple& tuple); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple); void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rangeSig& sig); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rangeSig& sig); void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::Bulletproof& p); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::Bulletproof& p); void fromJsonValue(const rapidjson::Value& val, rct::Bulletproof& p); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::boroSig& sig); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::boroSig& sig); void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::mgSig& sig); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig); void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::DaemonInfo& info); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info); -void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_distribution& dist); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); template <typename Map> -typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Map& map); +typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map); template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type fromJsonValue(const rapidjson::Value& val, Map& map); template <typename Vec> -typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Vec &vec); +typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec); template <typename Vec> typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec); @@ -315,7 +315,7 @@ typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJson // unfortunately because of how templates work they have to be here. template <typename Map> -inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Map& map) +inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map) { using key_type = typename Map::key_type; static_assert(std::is_same<std::string, key_type>() || is_to_hex<key_type>(), "invalid map key type"); @@ -351,7 +351,7 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from } template <typename Vec> -inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Vec &vec) +inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec) { dest.StartArray(); for (const auto& t : vec) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index d45ef3d7c..b1e69161f 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -143,6 +143,7 @@ enum TransferType { }; static std::string get_human_readable_timespan(std::chrono::seconds seconds); +static std::string get_human_readable_timespan(uint64_t seconds); namespace { @@ -182,6 +183,7 @@ namespace const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]"); const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_ACCOUNT("sweep_account <account> [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); @@ -207,7 +209,7 @@ namespace const char* USAGE_CHECK_SPEND_PROOF("check_spend_proof <txid> <signature_file> [<message>]"); const char* USAGE_GET_RESERVE_PROOF("get_reserve_proof (all|<amount>) [<message>]"); const char* USAGE_CHECK_RESERVE_PROOF("check_reserve_proof <address> <signature_file> [<message>]"); - const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); const char* USAGE_UNSPENT_OUTPUTS("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"); const char* USAGE_RESCAN_BC("rescan_bc [hard|soft|keep_ki] [start_height=0]"); const char* USAGE_SET_TX_NOTE("set_tx_note <txid> [free text note]"); @@ -3145,6 +3147,9 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), tr(USAGE_SWEEP_ALL), tr("Send all unlocked balance to an address. If the parameter \"index=<N1>[,<N2>,...]\" or \"index=all\" is specified, the wallet sweeps outputs received by those or all address indices, respectively. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and N > 0, wallet splits the transaction into N even outputs.")); + m_cmd_binder.set_handler("sweep_account", boost::bind(&simple_wallet::sweep_account, this, _1), + tr(USAGE_SWEEP_ACCOUNT), + tr("Send all unlocked balance from a given account to an address. If the parameter \"index=<N1>[,<N2>,...]\" or \"index=all\" is specified, the wallet sweeps outputs received by those or all address indices, respectively. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and N > 0, wallet splits the transaction into N even outputs.")); m_cmd_binder.set_handler("sweep_below", boost::bind(&simple_wallet::on_command, this, &simple_wallet::sweep_below, _1), tr(USAGE_SWEEP_BELOW), @@ -3338,7 +3343,7 @@ simple_wallet::simple_wallet() "** Set of address indices used as inputs in this transfer.")); m_cmd_binder.set_handler("export_transfers", boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_transfers, _1), - tr("export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"), + tr("export_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"), tr("Export to CSV the incoming/outgoing transfers within an optional height range.")); m_cmd_binder.set_handler("unspent_outputs", boost::bind(&simple_wallet::on_command, this, &simple_wallet::unspent_outputs, _1), @@ -5576,14 +5581,19 @@ boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request() return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device) +boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool & on_device) { - if (on_device){ - message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); - return boost::none; + if (on_device) { + std::string accepted = input_line(tr( + "Device asks for passphrase. Do you want to enter the passphrase on device (Y) (or on the host (N))?")); + if (std::cin.eof() || command_line::is_yes(accepted)) { + message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); + return boost::none; + } } PAUSE_READLINE(); + on_device = false; std::string msg = tr("Enter device passphrase"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); @@ -5748,15 +5758,19 @@ bool simple_wallet::show_balance_unlocked(bool detailed) success_msg_writer() << tr("Currently selected account: [") << m_current_subaddress_account << tr("] ") << m_wallet->get_subaddress_label({m_current_subaddress_account, 0}); const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); - uint64_t blocks_to_unlock; - uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, false, &blocks_to_unlock); + uint64_t blocks_to_unlock, time_to_unlock; + uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, false, &blocks_to_unlock, &time_to_unlock); std::string unlock_time_message; - if (blocks_to_unlock > 0) + if (blocks_to_unlock > 0 && time_to_unlock > 0) + unlock_time_message = (boost::format(" (%lu block(s) and %s to unlock)") % blocks_to_unlock % get_human_readable_timespan(time_to_unlock)).str(); + else if (blocks_to_unlock > 0) unlock_time_message = (boost::format(" (%lu block(s) to unlock)") % blocks_to_unlock).str(); + else if (time_to_unlock > 0) + unlock_time_message = (boost::format(" (%s to unlock)") % get_human_readable_timespan(time_to_unlock)).str(); success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance(m_current_subaddress_account, false)) << ", " << tr("unlocked balance: ") << print_money(unlocked_balance) << unlock_time_message << extra; std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(m_current_subaddress_account, false); - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account, false); + std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account, false); if (!detailed || balance_per_subaddress.empty()) return true; success_msg_writer() << tr("Balance per address:"); @@ -6221,6 +6235,7 @@ bool simple_wallet::prompt_if_old(const std::vector<tools::wallet2::pending_tx> } return true; } +//---------------------------------------------------------------------------------------------------- void simple_wallet::check_for_inactivity_lock(bool user) { if (m_locked) @@ -6745,7 +6760,7 @@ bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) { - sweep_main(0, true, args_); + sweep_main(m_current_subaddress_account, 0, true, args_); return true; } //---------------------------------------------------------------------------------------------------- @@ -6856,18 +6871,22 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args_) +bool simple_wallet::sweep_main(uint32_t account, uint64_t below, bool locked, const std::vector<std::string> &args_) { - auto print_usage = [below]() + auto print_usage = [this, account, below]() { if (below) { PRINT_USAGE(USAGE_SWEEP_BELOW); } - else + else if (account == m_current_subaddress_account) { PRINT_USAGE(USAGE_SWEEP_ALL); } + else + { + PRINT_USAGE(USAGE_SWEEP_ACCOUNT); + } }; if (args_.size() == 0) { @@ -7041,7 +7060,7 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, fake_outs_count, unlock_block /* unlock_time */, priority, extra, account, subaddr_indices); if (ptx_vector.empty()) { @@ -7381,7 +7400,27 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector<std::string> &args_) { - sweep_main(0, false, args_); + sweep_main(m_current_subaddress_account, 0, false, args_); + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::sweep_account(const std::vector<std::string> &args_) +{ + auto local_args = args_; + if (local_args.empty()) + { + PRINT_USAGE(USAGE_SWEEP_ACCOUNT); + return true; + } + uint32_t account = 0; + if (!epee::string_tools::get_xtype_from_string(account, local_args[0])) + { + fail_msg_writer() << tr("Invalid account"); + return true; + } + local_args.erase(local_args.begin()); + + sweep_main(account, 0, false, local_args); return true; } //---------------------------------------------------------------------------------------------------- @@ -7398,7 +7437,7 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); + sweep_main(m_current_subaddress_account, below, false, std::vector<std::string>(++args_.begin(), args_.end())); return true; } //---------------------------------------------------------------------------------------------------- @@ -8233,6 +8272,11 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds) return sw::tr("a long time"); } //---------------------------------------------------------------------------------------------------- +static std::string get_human_readable_timespan(uint64_t seconds) +{ + return get_human_readable_timespan(std::chrono::seconds(seconds)); +} +//---------------------------------------------------------------------------------------------------- // mutates local_args as it parses and consumes arguments bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vector<transfer_view>& transfers) { @@ -8495,7 +8539,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::vector<std::string> local_args = args_; if(local_args.size() > 4) { - fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); return true; } @@ -8548,7 +8592,7 @@ bool simple_wallet::export_transfers(const std::vector<std::string>& args_) std::vector<std::string> local_args = args_; if(local_args.size() > 5) { - fail_msg_writer() << tr("usage: export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<path>]"); + fail_msg_writer() << tr("usage: export_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<path>]"); return true; } diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 4ba2793e0..59818b303 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -170,8 +170,9 @@ namespace cryptonote bool transfer(const std::vector<std::string> &args); bool locked_transfer(const std::vector<std::string> &args); bool locked_sweep_all(const std::vector<std::string> &args); - bool sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args); + bool sweep_main(uint32_t account, uint64_t below, bool locked, const std::vector<std::string> &args); bool sweep_all(const std::vector<std::string> &args); + bool sweep_account(const std::vector<std::string> &args); bool sweep_below(const std::vector<std::string> &args); bool sweep_single(const std::vector<std::string> &args); bool sweep_unmixable(const std::vector<std::string> &args); @@ -348,7 +349,7 @@ namespace cryptonote virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); virtual void on_device_button_request(uint64_t code); virtual boost::optional<epee::wipeable_string> on_device_pin_request(); - virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device); //---------------------------------------------------------- friend class refresh_progress_reporter_t; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 4612b0397..d89261c64 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -267,13 +267,15 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback return boost::none; } - virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device) { if (m_listener) { auto passphrase = m_listener->onDevicePassphraseRequest(on_device); - if (!on_device && passphrase) { + if (passphrase) { return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size())); } + } else { + on_device = true; } return boost::none; } diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 09c64106e..9c3df8988 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -400,8 +400,8 @@ struct WalletListener /** * @brief called by device when passphrase entry is needed */ - virtual optional<std::string> onDevicePassphraseRequest(bool on_device) { - if (!on_device) throw std::runtime_error("Not supported"); + virtual optional<std::string> onDevicePassphraseRequest(bool & on_device) { + on_device = true; return optional<std::string>(); } @@ -1293,7 +1293,11 @@ struct WalletManager virtual std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const = 0; //! checks for an update and returns version, hash and url - static std::tuple<bool, std::string, std::string, std::string, std::string> checkUpdates(const std::string &software, std::string subdir); + static std::tuple<bool, std::string, std::string, std::string, std::string> checkUpdates( + const std::string &software, + std::string subdir, + const char *buildtag = nullptr, + const char *current_version = nullptr); }; diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 44a184304..8d7541cea 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -342,22 +342,30 @@ std::string WalletManagerImpl::resolveOpenAlias(const std::string &address, bool return addresses.front(); } -std::tuple<bool, std::string, std::string, std::string, std::string> WalletManager::checkUpdates(const std::string &software, std::string subdir) +std::tuple<bool, std::string, std::string, std::string, std::string> WalletManager::checkUpdates( + const std::string &software, + std::string subdir, + const char *buildtag/* = nullptr*/, + const char *current_version/* = nullptr*/) { + if (buildtag == nullptr) + { #ifdef BUILD_TAG - static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG); + static const char buildtag_default[] = BOOST_PP_STRINGIZE(BUILD_TAG); #else - static const char buildtag[] = "source"; - // Override the subdir string when built from source - subdir = "source"; + static const char buildtag_default[] = "source"; + // Override the subdir string when built from source + subdir = "source"; #endif + buildtag = buildtag_default; + } std::string version, hash; MDEBUG("Checking for a new " << software << " version for " << buildtag); if (!tools::check_updates(software, buildtag, version, hash)) return std::make_tuple(false, "", "", "", ""); - if (tools::vercmp(version.c_str(), MONERO_VERSION) > 0) + if (tools::vercmp(version.c_str(), current_version != nullptr ? current_version : MONERO_VERSION) > 0) { std::string user_url = tools::get_update_url(software, subdir, buildtag, version, true); std::string auto_url = tools::get_update_url(software, subdir, buildtag, version, false); diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 6e2cb933f..1bd462ef5 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -48,7 +48,7 @@ namespace mms { -message_store::message_store() +message_store::message_store(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client) : m_transporter(std::move(http_client)) { m_active = false; m_auto_send = false; diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h index 637bd29a1..d40daf186 100644 --- a/src/wallet/message_store.h +++ b/src/wallet/message_store.h @@ -43,6 +43,7 @@ #include "common/i18n.h" #include "common/command_line.h" #include "wipeable_string.h" +#include "net/abstract_http_client.h" #include "message_transporter.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -202,7 +203,8 @@ namespace mms class message_store { public: - message_store(); + message_store(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client); + // Initialize and start to use the MMS, set the first signer, this wallet itself // Filename, if not null and not empty, is used to create the ".mms" file // reset it if already used, with deletion of all signers and messages diff --git a/src/wallet/message_transporter.cpp b/src/wallet/message_transporter.cpp index cf9b45b37..4dd4b8f01 100644 --- a/src/wallet/message_transporter.cpp +++ b/src/wallet/message_transporter.cpp @@ -80,7 +80,7 @@ namespace bitmessage_rpc } -message_transporter::message_transporter() +message_transporter::message_transporter(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client) : m_http_client(std::move(http_client)) { m_run = true; } @@ -96,7 +96,7 @@ void message_transporter::set_options(const std::string &bitmessage_address, con } m_bitmessage_login = bitmessage_login; - m_http_client.set_server(address_parts.host, std::to_string(address_parts.port), boost::none); + m_http_client->set_server(address_parts.host, std::to_string(address_parts.port), boost::none); } bool message_transporter::receive_messages(const std::vector<std::string> &destination_transport_addresses, @@ -256,7 +256,7 @@ bool message_transporter::post_request(const std::string &request, std::string & additional_params.push_back(std::make_pair("Content-Type", "application/xml; charset=utf-8")); const epee::net_utils::http::http_response_info* response = NULL; std::chrono::milliseconds timeout = std::chrono::seconds(15); - bool r = m_http_client.invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params)); + bool r = m_http_client->invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params)); if (r) { answer = response->m_body; @@ -266,7 +266,7 @@ bool message_transporter::post_request(const std::string &request, std::string & LOG_ERROR("POST request to Bitmessage failed: " << request.substr(0, 300)); THROW_WALLET_EXCEPTION(tools::error::no_connection_to_bitmessage, m_bitmessage_url); } - m_http_client.disconnect(); // see comment above + m_http_client->disconnect(); // see comment above std::string string_value = get_str_between_tags(answer, "<string>", "</string>"); if ((string_value.find("API Error") == 0) || (string_value.find("RPC ") == 0)) { diff --git a/src/wallet/message_transporter.h b/src/wallet/message_transporter.h index 28c099d87..84a2e9bae 100644 --- a/src/wallet/message_transporter.h +++ b/src/wallet/message_transporter.h @@ -34,9 +34,9 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "net/http_server_impl_base.h" #include "net/http_client.h" +#include "net/abstract_http_client.h" #include "common/util.h" #include "wipeable_string.h" -#include "serialization/keyvalue_serialization.h" #include <vector> namespace mms @@ -83,7 +83,7 @@ typedef epee::misc_utils::struct_init<transport_message_t> transport_message; class message_transporter { public: - message_transporter(); + message_transporter(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client); void set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login); bool send_message(const transport_message &message); bool receive_messages(const std::vector<std::string> &destination_transport_addresses, @@ -94,7 +94,7 @@ public: bool delete_transport_address(const std::string &transport_address); private: - epee::net_utils::http::http_simple_client m_http_client; + const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client; std::string m_bitmessage_url; epee::wipeable_string m_bitmessage_login; std::atomic<bool> m_run; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index f3698b599..873c2ee51 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -51,7 +51,7 @@ namespace tools static const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); -NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex) +NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::abstract_http_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex) : m_http_client(http_client) , m_rpc_payment_state(rpc_payment_state) , m_daemon_rpc_mutex(mutex) diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 65ca40640..b053659e9 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -31,7 +31,7 @@ #include <string> #include <boost/thread/mutex.hpp> #include "include_base_utils.h" -#include "net/http_client.h" +#include "net/abstract_http_client.h" #include "rpc/core_rpc_server_commands_defs.h" #include "wallet_rpc_helpers.h" @@ -41,7 +41,7 @@ namespace tools class NodeRPCProxy { public: - NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex); + NodeRPCProxy(epee::net_utils::http::abstract_http_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex); void set_client_secret_key(const crypto::secret_key &skey) { m_client_id_secret_key = skey; } void invalidate(); @@ -72,7 +72,7 @@ private: private: boost::optional<std::string> get_info(); - epee::net_utils::http::http_simple_client &m_http_client; + epee::net_utils::http::abstract_http_client &m_http_client; rpc_payment_state_t &m_rpc_payment_state; boost::recursive_mutex &m_daemon_rpc_mutex; crypto::secret_key m_client_id_secret_key; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index bc8219c69..a9f454209 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1109,10 +1109,12 @@ boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request() return boost::none; } -boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device) +boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool & on_device) { if (wallet) return wallet->on_device_passphrase_request(on_device); + else + on_device = true; return boost::none; } @@ -1122,7 +1124,8 @@ void wallet_device_callback::on_progress(const hw::device_progress& event) wallet->on_device_progress(event); } -wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): +wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory): + m_http_client(std::move(http_client_factory->create())), m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), m_upper_transaction_weight_limit(0), @@ -1167,7 +1170,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_watch_only(false), m_multisig(false), m_multisig_threshold(0), - m_node_rpc_proxy(m_http_client, m_rpc_payment_state, m_daemon_rpc_mutex), + m_node_rpc_proxy(*m_http_client, m_rpc_payment_state, m_daemon_rpc_mutex), m_account_public_address{crypto::null_pkey, crypto::null_pkey}, m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR), @@ -1178,7 +1181,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_light_wallet_balance(0), m_light_wallet_unlocked_balance(0), m_original_keys_available(false), - m_message_store(), + m_message_store(http_client_factory->create()), m_key_device_type(hw::device::device_type::SOFTWARE), m_ring_history_saved(false), m_ringdb(), @@ -1298,8 +1301,8 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u { boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - if(m_http_client.is_connected()) - m_http_client.disconnect(); + if(m_http_client->is_connected()) + m_http_client->disconnect(); const bool changed = m_daemon_address != daemon_address; m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); @@ -1313,7 +1316,7 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u const std::string address = get_daemon_address(); MINFO("setting daemon to " << address); - bool ret = m_http_client.set_server(address, get_daemon_login(), std::move(ssl_options)); + bool ret = m_http_client->set_server(address, get_daemon_login(), std::move(ssl_options)); if (ret) { CRITICAL_REGION_LOCAL(default_daemon_address_lock); @@ -1328,7 +1331,12 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_is_initialized = true; m_upper_transaction_weight_limit = upper_transaction_weight_limit; if (proxy != boost::asio::ip::tcp::endpoint{}) - m_http_client.set_connector(net::socks::connector{std::move(proxy)}); + { + epee::net_utils::http::abstract_http_client* abstract_http_client = m_http_client.get(); + epee::net_utils::http::http_simple_client* http_simple_client = dynamic_cast<epee::net_utils::http::http_simple_client*>(abstract_http_client); + CHECK_AND_ASSERT_MES(http_simple_client != nullptr, false, "http_simple_client must be used to set proxy"); + http_simple_client->set_connector(net::socks::connector{std::move(proxy)}); + } return set_daemon(daemon_address, daemon_login, trusted_daemon, std::move(ssl_options)); } //---------------------------------------------------------------------------------------------------- @@ -2593,7 +2601,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "getblocks.bin", error::get_blocks_error, get_rpc_status(res.status)); THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" + @@ -2622,7 +2630,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; req.client = get_client_signature(); uint64_t pre_call_credits = m_rpc_payment_state.credits; - bool r = net_utils::invoke_http_bin("/gethashes.bin", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_bin("/gethashes.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "gethashes.bin", error::get_hashes_error, get_rpc_status(res.status)); check_rpc_cost("/gethashes.bin", res.credits, pre_call_credits, 1 + res.m_block_ids.size() * COST_PER_BLOCK_HASH); } @@ -2907,7 +2915,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_transaction_pool_hashes.bin", error::get_tx_pool_error); check_rpc_cost("/get_transaction_pool_hashes.bin", res.credits, pre_call_credits, 1 + res.tx_hashes.size() * COST_PER_POOL_HASH); } @@ -3052,7 +3060,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); if (r && res.status == CORE_RPC_STATUS_OK) check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX); } @@ -3538,7 +3546,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, m_http_client, rpc_timeout); + r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/get_output_distribution.bin"); check_rpc_cost("/get_output_distribution.bin", res.credits, pre_call_credits, COST_PER_OUTPUT_DISTRIBUTION_0); } @@ -3697,6 +3705,30 @@ void wallet2::clear_soft(bool keep_key_images) */ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only) { + boost::optional<wallet2::keys_file_data> keys_file_data = get_keys_file_data(password, watch_only); + CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data"); + + std::string tmp_file_name = keys_file_name + ".new"; + std::string buf; + bool r = ::serialization::dump_binary(keys_file_data.get(), buf); + r = r && save_to_file(tmp_file_name, buf); + CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); + + unlock_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); + lock_keys_file(); + + if (e) { + boost::filesystem::remove(tmp_file_name); + LOG_ERROR("failed to update wallet keys file " << keys_file_name); + return false; + } + + return true; +} +//---------------------------------------------------------------------------------------------------- +boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only) +{ std::string account_data; std::string multisig_signers; std::string multisig_derivations; @@ -3717,8 +3749,8 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account.encrypt_keys(key); bool r = epee::serialization::store_t_to_binary(account, account_data); - CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys"); - wallet2::keys_file_data keys_file_data = {}; + CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet keys"); + boost::optional<wallet2::keys_file_data> keys_file_data = (wallet2::keys_file_data) {}; // Create a JSON object with "key_data" and "seed_language" as keys. rapidjson::Document json; @@ -3749,12 +3781,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable if (m_multisig) { bool r = ::serialization::dump_binary(m_multisig_signers, multisig_signers); - CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig signers"); + CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet multisig signers"); value.SetString(multisig_signers.c_str(), multisig_signers.length()); json.AddMember("multisig_signers", value, json.GetAllocator()); r = ::serialization::dump_binary(m_multisig_derivations, multisig_derivations); - CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig derivations"); + CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet multisig derivations"); value.SetString(multisig_derivations.c_str(), multisig_derivations.length()); json.AddMember("multisig_derivations", value, json.GetAllocator()); @@ -3897,27 +3929,10 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable // Encrypt the entire JSON object. std::string cipher; cipher.resize(account_data.size()); - keys_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); - keys_file_data.account_data = cipher; - - std::string tmp_file_name = keys_file_name + ".new"; - std::string buf; - r = ::serialization::dump_binary(keys_file_data, buf); - r = r && save_to_file(tmp_file_name, buf); - CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); - - unlock_keys_file(); - std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); - lock_keys_file(); - - if (e) { - boost::filesystem::remove(tmp_file_name); - LOG_ERROR("failed to update wallet keys file " << keys_file_name); - return false; - } - - return true; + keys_file_data.get().iv = crypto::rand<crypto::chacha_iv>(); + crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.get().iv, &cipher[0]); + keys_file_data.get().account_data = cipher; + return keys_file_data; } //---------------------------------------------------------------------------------------------------- void wallet2::setup_keys(const epee::wipeable_string &password) @@ -3957,16 +3972,51 @@ void wallet2::change_password(const std::string &filename, const epee::wipeable_ */ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_string& password) { - rapidjson::Document json; - wallet2::keys_file_data keys_file_data; - std::string buf; - bool encrypted_secret_keys = false; - bool r = load_from_file(keys_file_name, buf); + std::string keys_file_buf; + bool r = load_from_file(keys_file_name, keys_file_buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); + // Load keys from buffer + boost::optional<crypto::chacha_key> keys_to_encrypt; + try { + r = wallet2::load_keys_buf(keys_file_buf, password, keys_to_encrypt); + } catch (const std::exception& e) { + std::size_t found = string(e.what()).find("failed to deserialize keys buffer"); + THROW_WALLET_EXCEPTION_IF(found != std::string::npos, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + throw e; + } + + // Rewrite with encrypted keys if unencrypted, ignore errors + if (r && keys_to_encrypt != boost::none) + { + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + encrypt_keys(keys_to_encrypt.get()); + bool saved_ret = store_keys(keys_file_name, password, m_watch_only); + if (!saved_ret) + { + // just moan a bit, but not fatal + MERROR("Error saving keys file with encrypted keys, not fatal"); + } + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + decrypt_keys(keys_to_encrypt.get()); + m_keys_file_locker.reset(); + } + return r; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password) { + boost::optional<crypto::chacha_key> keys_to_encrypt; + return wallet2::load_keys_buf(keys_buf, password, keys_to_encrypt); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt) { + // Decrypt the contents - r = ::serialization::parse_binary(buf, keys_file_data); - THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + rapidjson::Document json; + wallet2::keys_file_data keys_file_data; + bool encrypted_secret_keys = false; + bool r = ::serialization::parse_binary(keys_buf, keys_file_data); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize keys buffer"); crypto::chacha_key key; crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string account_data; @@ -4250,8 +4300,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ } else { - THROW_WALLET_EXCEPTION(error::wallet_internal_error, "invalid password"); - return false; + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "invalid password"); + return false; } r = epee::serialization::load_t_from_binary(m_account, account_data); @@ -4285,24 +4335,13 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ } else { - // rewrite with encrypted keys, ignore errors - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) - encrypt_keys(key); - bool saved_ret = store_keys(keys_file_name, password, m_watch_only); - if (!saved_ret) - { - // just moan a bit, but not fatal - MERROR("Error saving keys file with encrypted keys, not fatal"); - } - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) - decrypt_keys(key); - m_keys_file_locker.reset(); + keys_to_encrypt = key; } } const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if(!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); @@ -4809,6 +4848,7 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, std::vector<crypto::secret_key> multisig_keys; rct::key spend_pkey = rct::identity(); rct::key spend_skey; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(&spend_skey, sizeof(spend_skey));}); std::vector<crypto::public_key> multisig_signers; // decrypt keys @@ -4921,7 +4961,8 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, // re-encrypt keys keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); - create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + if (!m_wallet_file.empty()) + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); setup_new_blockchain(); @@ -5061,7 +5102,9 @@ std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &passwor ++m_multisig_rounds_passed; - create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + if (!m_wallet_file.empty()) + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + return extra_multisig_info; } @@ -5435,13 +5478,13 @@ bool wallet2::check_connection(uint32_t *version, bool *ssl, uint32_t timeout) { boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - if(!m_http_client.is_connected(ssl)) + if(!m_http_client->is_connected(ssl)) { m_rpc_version = 0; m_node_rpc_proxy.invalidate(); - if (!m_http_client.connect(std::chrono::milliseconds(timeout))) + if (!m_http_client->connect(std::chrono::milliseconds(timeout))) return false; - if(!m_http_client.is_connected(ssl)) + if(!m_http_client->is_connected(ssl)) return false; } } @@ -5469,12 +5512,12 @@ void wallet2::set_offline(bool offline) { m_offline = offline; m_node_rpc_proxy.set_offline(offline); - m_http_client.set_auto_connect(!offline); + m_http_client->set_auto_connect(!offline); if (offline) { boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - if(m_http_client.is_connected()) - m_http_client.disconnect(); + if(m_http_client->is_connected()) + m_http_client->disconnect(); } } //---------------------------------------------------------------------------------------------------- @@ -5489,48 +5532,63 @@ void wallet2::generate_chacha_key_from_password(const epee::wipeable_string &pas crypto::generate_chacha_key(pass.data(), pass.size(), key, m_kdf_rounds); } //---------------------------------------------------------------------------------------------------- -void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password) +void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password, const std::string& keys_buf, const std::string& cache_buf) { clear(); prepare_file_names(wallet_); + // determine if loading from file system or string buffer + bool use_fs = !wallet_.empty(); + THROW_WALLET_EXCEPTION_IF((use_fs && !keys_buf.empty()) || (!use_fs && keys_buf.empty()), error::file_read_error, "must load keys either from file system or from buffer");\ + boost::system::error_code e; - bool exists = boost::filesystem::exists(m_keys_file, e); - THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); - lock_keys_file(); - THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); + if (use_fs) + { + bool exists = boost::filesystem::exists(m_keys_file, e); + THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); + lock_keys_file(); + THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); - // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). - unlock_keys_file(); - if (!load_keys(m_keys_file, password)) + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); + if (!load_keys(m_keys_file, password)) + { + THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); + } + LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); + lock_keys_file(); + } + else if (!load_keys_buf(keys_buf, password)) { - THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); + THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, "failed to load keys from buffer"); } - LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); - lock_keys_file(); wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password); //keys loaded ok! //try to load wallet file. but even if we failed, it is not big problem - if(!boost::filesystem::exists(m_wallet_file, e) || e) + if (use_fs && (!boost::filesystem::exists(m_wallet_file, e) || e)) { LOG_PRINT_L0("file not found: " << m_wallet_file << ", starting with empty blockchain"); m_account_public_address = m_account.get_keys().m_account_address; } - else + else if (use_fs || !cache_buf.empty()) { wallet2::cache_file_data cache_file_data; - std::string buf; - bool r = load_from_file(m_wallet_file, buf, std::numeric_limits<size_t>::max()); - THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); + std::string cache_file_buf; + bool r = true; + if (use_fs) + { + load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits<size_t>::max()); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); + } // try to read it as an encrypted cache try { LOG_PRINT_L1("Trying to decrypt cache data"); - r = ::serialization::parse_binary(buf, cache_file_data); + r = ::serialization::parse_binary(use_fs ? cache_file_buf : cache_buf, cache_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); @@ -5567,7 +5625,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass catch (...) { LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + if (use_fs) boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); std::stringstream iss; iss.str(""); iss << cache_data; @@ -5582,17 +5640,17 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass LOG_PRINT_L1("Failed to load encrypted cache, trying unencrypted"); try { std::stringstream iss; - iss << buf; + iss << cache_file_buf; boost::archive::portable_binary_iarchive ar(iss); ar >> *this; } catch (...) { LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + if (use_fs) boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); std::stringstream iss; iss.str(""); - iss << buf; + iss << cache_file_buf; boost::archive::binary_iarchive ar(iss); ar >> *this; } @@ -5636,7 +5694,8 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass try { - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + if (use_fs) + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); } catch (const std::exception &e) { @@ -5664,7 +5723,7 @@ void wallet2::trim_hashchain() req.height = m_blockchain.size() - 1; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, m_http_client, rpc_timeout); + r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, *m_http_client, rpc_timeout); if (r && res.status == CORE_RPC_STATUS_OK) check_rpc_cost("getblockheaderbyheight", res.credits, pre_call_credits, COST_PER_BLOCK_HEADER); } @@ -5739,18 +5798,10 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas } } } - // preparing wallet data - std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << *this; - wallet2::cache_file_data cache_file_data = {}; - cache_file_data.cache_data = oss.str(); - std::string cipher; - cipher.resize(cache_file_data.cache_data.size()); - cache_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cipher[0]); - cache_file_data.cache_data = cipher; + // get wallet cache data + boost::optional<wallet2::cache_file_data> cache_file_data = get_cache_file_data(password); + THROW_WALLET_EXCEPTION_IF(cache_file_data == boost::none, error::wallet_internal_error, "failed to generate wallet cache data"); const std::string new_file = same_file ? m_wallet_file + ".new" : path; const std::string old_file = m_wallet_file; @@ -5801,7 +5852,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas // The price to pay is temporary higher memory consumption for string stream + binary archive std::ostringstream oss; binary_archive<true> oar(oss); - bool success = ::serialization::serialize(oar, cache_file_data); + bool success = ::serialization::serialize(oar, cache_file_data.get()); if (success) { success = save_to_file(new_file, oss.str()); } @@ -5810,7 +5861,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas std::ofstream ostr; ostr.open(new_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); binary_archive<true> oar(ostr); - bool success = ::serialization::serialize(oar, cache_file_data); + bool success = ::serialization::serialize(oar, cache_file_data.get()); ostr.close(); THROW_WALLET_EXCEPTION_IF(!success || !ostr.good(), error::file_save_error, new_file); #endif @@ -5826,7 +5877,30 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas // store should only exist if the MMS is really active m_message_store.write_to_file(get_multisig_wallet_state(), m_mms_file); } - +} +//---------------------------------------------------------------------------------------------------- +boost::optional<wallet2::cache_file_data> wallet2::get_cache_file_data(const epee::wipeable_string &passwords) +{ + trim_hashchain(); + try + { + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << *this; + + boost::optional<wallet2::cache_file_data> cache_file_data = (wallet2::cache_file_data) {}; + cache_file_data.get().cache_data = oss.str(); + std::string cipher; + cipher.resize(cache_file_data.get().cache_data.size()); + cache_file_data.get().iv = crypto::rand<crypto::chacha_iv>(); + crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), m_cache_key, cache_file_data.get().iv, &cipher[0]); + cache_file_data.get().cache_data = cipher; + return cache_file_data; + } + catch(...) + { + return boost::none; + } } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::balance(uint32_t index_major, bool strict) const @@ -5839,18 +5913,22 @@ uint64_t wallet2::balance(uint32_t index_major, bool strict) const return amount; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t *blocks_to_unlock) const +uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t *blocks_to_unlock, uint64_t *time_to_unlock) const { uint64_t amount = 0; if (blocks_to_unlock) *blocks_to_unlock = 0; + if (time_to_unlock) + *time_to_unlock = 0; if(m_light_wallet) return m_light_wallet_balance; for (const auto& i : unlocked_balance_per_subaddress(index_major, strict)) { amount += i.second.first; - if (blocks_to_unlock && i.second.second > *blocks_to_unlock) - *blocks_to_unlock = i.second.second; + if (blocks_to_unlock && i.second.second.first > *blocks_to_unlock) + *blocks_to_unlock = i.second.second.first; + if (time_to_unlock && i.second.second.second > *time_to_unlock) + *time_to_unlock = i.second.second.second; } return amount; } @@ -5887,35 +5965,40 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo return amount_per_subaddr; } //---------------------------------------------------------------------------------------------------- -std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major, bool strict) const +std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major, bool strict) const { - std::map<uint32_t, std::pair<uint64_t, uint64_t>> amount_per_subaddr; + std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> amount_per_subaddr; const uint64_t blockchain_height = get_blockchain_current_height(); + const uint64_t now = time(NULL); for(const transfer_details& td: m_transfers) { if(td.m_subaddr_index.major == index_major && !is_spent(td, strict) && !td.m_frozen) { - uint64_t amount = 0, blocks_to_unlock = 0; + uint64_t amount = 0, blocks_to_unlock = 0, time_to_unlock = 0; if (is_transfer_unlocked(td)) { amount = td.amount(); blocks_to_unlock = 0; + time_to_unlock = 0; } else { uint64_t unlock_height = td.m_block_height + std::max<uint64_t>(CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS); if (td.m_tx.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && td.m_tx.unlock_time > unlock_height) unlock_height = td.m_tx.unlock_time; + uint64_t unlock_time = td.m_tx.unlock_time >= CRYPTONOTE_MAX_BLOCK_NUMBER ? td.m_tx.unlock_time : 0; blocks_to_unlock = unlock_height > blockchain_height ? unlock_height - blockchain_height : 0; + time_to_unlock = unlock_time > now ? unlock_time - now : 0; amount = 0; } auto found = amount_per_subaddr.find(td.m_subaddr_index.minor); if (found == amount_per_subaddr.end()) - amount_per_subaddr[td.m_subaddr_index.minor] = std::make_pair(amount, blocks_to_unlock); + amount_per_subaddr[td.m_subaddr_index.minor] = std::make_pair(amount, std::make_pair(blocks_to_unlock, time_to_unlock)); else { found->second.first += amount; - found->second.second = std::max(found->second.second, blocks_to_unlock); + found->second.second.first = std::max(found->second.second.first, blocks_to_unlock); + found->second.second.second = std::max(found->second.second.second, time_to_unlock); } } } @@ -5930,17 +6013,21 @@ uint64_t wallet2::balance_all(bool strict) const return r; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock) const +uint64_t wallet2::unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock, uint64_t *time_to_unlock) const { uint64_t r = 0; if (blocks_to_unlock) *blocks_to_unlock = 0; + if (time_to_unlock) + *time_to_unlock = 0; for (uint32_t index_major = 0; index_major < get_num_subaddress_accounts(); ++index_major) { - uint64_t local_blocks_to_unlock; - r += unlocked_balance(index_major, strict, blocks_to_unlock ? &local_blocks_to_unlock : NULL); + uint64_t local_blocks_to_unlock, local_time_to_unlock; + r += unlocked_balance(index_major, strict, blocks_to_unlock ? &local_blocks_to_unlock : NULL, time_to_unlock ? &local_time_to_unlock : NULL); if (blocks_to_unlock) *blocks_to_unlock = std::max(*blocks_to_unlock, local_blocks_to_unlock); + if (time_to_unlock) + *time_to_unlock = std::max(*time_to_unlock, local_time_to_unlock); } return r; } @@ -6030,7 +6117,7 @@ void wallet2::rescan_spent() const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "is_key_image_spent", error::is_key_image_spent_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + @@ -6359,7 +6446,7 @@ void wallet2::commit_tx(pending_tx& ptx) oreq.tx = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx)); { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - bool r = epee::net_utils::invoke_http_json("/submit_raw_tx", oreq, ores, m_http_client, rpc_timeout, "POST"); + bool r = epee::net_utils::invoke_http_json("/submit_raw_tx", oreq, ores, *m_http_client, rpc_timeout, "POST"); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "submit_raw_tx"); // MyMonero and OpenMonero use different status strings THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, get_rpc_status(ores.status), ores.error); @@ -6378,7 +6465,7 @@ void wallet2::commit_tx(pending_tx& ptx) const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/sendrawtransaction", req, daemon_send_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/sendrawtransaction", req, daemon_send_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_send_resp, "sendrawtransaction", error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp)); check_rpc_cost("/sendrawtransaction", daemon_send_resp.credits, pre_call_credits, COST_PER_TX_RELAY); } @@ -6419,7 +6506,7 @@ void wallet2::commit_tx(pending_tx& ptx) // tx generated, get rid of used k values for (size_t idx: ptx.selected_transfers) - m_transfers[idx].m_multisig_k.clear(); + memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0])); //fee includes dust if dust policy specified it. LOG_PRINT_L1("Transaction successfully sent. <" << txid << ">" << ENDL @@ -6861,13 +6948,13 @@ std::string wallet2::save_multisig_tx(multisig_tx_set txs) // txes generated, get rid of used k values for (size_t n = 0; n < txs.m_ptx.size(); ++n) for (size_t idx: txs.m_ptx[n].construction_data.selected_transfers) - m_transfers[idx].m_multisig_k.clear(); + memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0])); // zero out some data we don't want to share for (auto &ptx: txs.m_ptx) { for (auto &e: ptx.construction_data.sources) - e.multisig_kLRki.k = rct::zero(); + memwipe(&e.multisig_kLRki.k, sizeof(e.multisig_kLRki.k)); } for (auto &ptx: txs.m_ptx) @@ -7075,10 +7162,12 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto ptx.tx.rct_signatures = sig.sigs; rct::keyV k; + rct::key skey = rct::zero(); + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){ memwipe(k.data(), k.size() * sizeof(k[0])); memwipe(&skey, sizeof(skey)); }); + for (size_t idx: sd.selected_transfers) k.push_back(get_multisig_k(idx, sig.used_L)); - rct::key skey = rct::zero(); for (const auto &msk: get_account().get_multisig_keys()) { crypto::public_key pmsk = get_multisig_signing_public_key(msk); @@ -7126,7 +7215,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto // txes generated, get rid of used k values for (size_t n = 0; n < exported_txs.m_ptx.size(); ++n) for (size_t idx: exported_txs.m_ptx[n].construction_data.selected_transfers) - m_transfers[idx].m_multisig_k.clear(); + memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0])); exported_txs.m_signers.insert(get_multisig_signer_public_key()); @@ -7350,7 +7439,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; getbh_req.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, getbh_res, "getblockheadersrange", error::get_blocks_error, get_rpc_status(getbh_res.status)); check_rpc_cost("/sendrawtransaction", getbh_res.credits, pre_call_credits, N * COST_PER_BLOCK_HEADER); } @@ -7576,7 +7665,7 @@ bool wallet2::find_and_save_rings(bool force) const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -7724,7 +7813,7 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, m_http_client, rpc_timeout, "POST"); + bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, *m_http_client, rpc_timeout, "POST"); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_random_outs"); THROW_WALLET_EXCEPTION_IF(ores.amount_outs.empty() , error::wallet_internal_error, "No outputs received from light wallet node. Error: " + ores.Error); @@ -7911,7 +8000,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, get_rpc_status(resp_t.status)); check_rpc_cost("get_output_histogram", resp_t.credits, pre_call_credits, COST_PER_OUTPUT_HISTOGRAM * req_t.amounts.size()); } @@ -7937,7 +8026,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, *m_http_client, rpc_timeout * 1000); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_distribution", error::get_output_distribution, get_rpc_status(resp_t.status)); uint64_t expected_cost = 0; for (uint64_t amount: req_t.amounts) expected_cost += (amount ? COST_PER_OUTPUT_DISTRIBUTION : COST_PER_OUTPUT_DISTRIBUTION_0); @@ -8291,7 +8380,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "get_outs.bin", error::get_outs_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + @@ -9579,7 +9668,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // throw if attempting a transaction with no money THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination); - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); + std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false); if (subaddr_indices.empty()) // "index=<N1>[,<N2>,...]" wasn't specified -> use all the indices with non-zero unlocked balance @@ -10500,7 +10589,7 @@ uint8_t wallet2::get_current_hard_fork() m_daemon_rpc_mutex.lock(); req_t.version = 0; - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, *m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, tools::error::no_connection_to_daemon, "hard_fork_info"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "hard_fork_info"); @@ -10595,7 +10684,7 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, resp_t.status); uint64_t cost = req_t.amounts.empty() ? COST_PER_FULL_OUTPUT_HISTOGRAM : (COST_PER_OUTPUT_HISTOGRAM * req_t.amounts.size()); check_rpc_cost("get_output_histogram", resp_t.credits, pre_call_credits, cost); @@ -10637,7 +10726,7 @@ uint64_t wallet2::get_num_rct_outputs() const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, resp_t.status); THROW_WALLET_EXCEPTION_IF(resp_t.histogram.size() != 1, error::get_histogram_error, "Expected exactly one response"); THROW_WALLET_EXCEPTION_IF(resp_t.histogram[0].amount != 0, error::get_histogram_error, "Expected 0 amount"); @@ -10768,7 +10857,7 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; req.client = get_client_signature(); uint64_t pre_call_credits = m_rpc_payment_state.credits; - bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX); @@ -10821,7 +10910,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -10874,7 +10963,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -10938,7 +11027,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_outs.bin", error::get_outs_error, res.status); THROW_WALLET_EXCEPTION_IF(res.outs.size() != ring_size, error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + @@ -10996,7 +11085,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -11071,7 +11160,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_outs.bin", error::get_outs_error, res.status); THROW_WALLET_EXCEPTION_IF(res.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + @@ -11173,7 +11262,7 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX); @@ -11228,7 +11317,7 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + ok = net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX); @@ -11389,7 +11478,7 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + ok = net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX); @@ -11686,7 +11775,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; gettx_req.client = get_client_signature(); - bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client); + bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || gettx_res.txs.size() != proofs.size(), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", gettx_res.credits, pre_call_credits, gettx_res.txs.size() * COST_PER_TX); @@ -11703,7 +11792,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; kispent_req.client = get_client_signature(); - ok = epee::net_utils::invoke_http_json("/is_key_image_spent", kispent_req, kispent_res, m_http_client, rpc_timeout); + ok = epee::net_utils::invoke_http_json("/is_key_image_spent", kispent_req, kispent_res, *m_http_client, rpc_timeout); THROW_WALLET_EXCEPTION_IF(!ok || kispent_res.spent_status.size() != proofs.size(), error::wallet_internal_error, "Failed to get key image spent status from daemon"); check_rpc_cost("/is_key_image_spent", kispent_res.credits, pre_call_credits, kispent_res.spent_status.size() * COST_PER_KEY_IMAGE); @@ -12277,7 +12366,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, daemon_resp, "is_key_image_spent"); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != signed_key_images.size(), error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + @@ -12366,7 +12455,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; gettxs_req.client = get_client_signature(); uint64_t pre_call_credits = m_rpc_payment_state.credits; - bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, gettxs_res, "gettransactions"); THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size())); @@ -12877,7 +12966,7 @@ cryptonote::blobdata wallet2::export_multisig() { transfer_details &td = m_transfers[n]; crypto::key_image ki; - td.m_multisig_k.clear(); + memwipe(td.m_multisig_k.data(), td.m_multisig_k.size() * sizeof(td.m_multisig_k[0])); info[n].m_LR.clear(); info[n].m_partial_key_images.clear(); @@ -12986,6 +13075,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) CHECK_AND_ASSERT_THROW_MES(info.size() + 1 <= m_multisig_signers.size() && info.size() + 1 >= m_multisig_threshold, "Wrong number of multisig sources"); std::vector<std::vector<rct::key>> k; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(k.data(), k.size() * sizeof(k[0]));}); k.reserve(m_transfers.size()); for (const auto &td: m_transfers) k.push_back(td.m_multisig_k); @@ -13307,7 +13397,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, m_http_client, rpc_timeout); + r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, *m_http_client, rpc_timeout); if (r && res.status == CORE_RPC_STATUS_OK) check_rpc_cost("/getblocks_by_height.bin", res.credits, pre_call_credits, 3 * COST_PER_BLOCK); } @@ -13385,7 +13475,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_txpool_backlog", error::get_tx_pool_error); check_rpc_cost("get_txpool_backlog", res.credits, pre_call_credits, COST_PER_TX_POOL_STATS * res.backlog.size()); } @@ -13547,10 +13637,12 @@ boost::optional<epee::wipeable_string> wallet2::on_device_pin_request() return boost::none; } //---------------------------------------------------------------------------------------------------- -boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device) +boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool & on_device) { if (nullptr != m_callback) return m_callback->on_device_passphrase_request(on_device); + else + on_device = true; return boost::none; } //---------------------------------------------------------------------------------------------------- @@ -13724,12 +13816,12 @@ void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const c //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_bytes_sent() const { - return m_http_client.get_bytes_sent(); + return m_http_client->get_bytes_sent(); } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_bytes_received() const { - return m_http_client.get_bytes_received(); + return m_http_client->get_bytes_received(); } //---------------------------------------------------------------------------------------------------- std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only) @@ -13742,7 +13834,7 @@ std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only) { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - bool r = epee::net_utils::invoke_http_json("/get_public_nodes", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/get_public_nodes", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/get_public_nodes"); } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 7620d09d8..ba4377bd2 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -145,7 +145,7 @@ private: virtual void on_device_button_request(uint64_t code) {} virtual void on_device_button_pressed() {} virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; } - virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device) { on_device = true; return boost::none; } virtual void on_device_progress(const hw::device_progress& event) {}; // Common callbacks virtual void on_pool_tx_removed(const crypto::hash &txid) {} @@ -159,7 +159,7 @@ private: void on_button_request(uint64_t code=0) override; void on_button_pressed() override; boost::optional<epee::wipeable_string> on_pin_request() override; - boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) override; void on_progress(const hw::device_progress& event) override; private: wallet2 * wallet; @@ -269,7 +269,7 @@ private: static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1); - wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false); + wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory = std::unique_ptr<epee::net_utils::http::http_simple_client_factory>(new epee::net_utils::http::http_simple_client_factory())); ~wallet2(); struct multisig_info @@ -708,7 +708,7 @@ private: */ void rewrite(const std::string& wallet_name, const epee::wipeable_string& password); void write_watch_only_wallet(const std::string& wallet_name, const epee::wipeable_string& password, std::string &new_keys_filename); - void load(const std::string& wallet, const epee::wipeable_string& password); + void load(const std::string& wallet, const epee::wipeable_string& password, const std::string& keys_buf = "", const std::string& cache_buf = ""); void store(); /*! * \brief store_to Stores wallet to another file(s), deleting old ones @@ -716,6 +716,19 @@ private: * \param password Password to protect new wallet (TODO: probably better save the password in the wallet object?) */ void store_to(const std::string &path, const epee::wipeable_string &password); + /*! + * \brief get_keys_file_data Get wallet keys data which can be stored to a wallet file. + * \param password Password of the encrypted wallet buffer (TODO: probably better save the password in the wallet object?) + * \param watch_only true to include only view key, false to include both spend and view keys + * \return Encrypted wallet keys data which can be stored to a wallet file + */ + boost::optional<wallet2::keys_file_data> get_keys_file_data(const epee::wipeable_string& password, bool watch_only); + /*! + * \brief get_cache_file_data Get wallet cache data which can be stored to a wallet file. + * \param password Password to protect the wallet cache data (TODO: probably better save the password in the wallet object?) + * \return Encrypted wallet cache data which can be stored to a wallet file + */ + boost::optional<wallet2::cache_file_data> get_cache_file_data(const epee::wipeable_string& password); std::string path() const; @@ -822,13 +835,13 @@ private: // locked & unlocked balance of given or current subaddress account uint64_t balance(uint32_t subaddr_index_major, bool strict) const; - uint64_t unlocked_balance(uint32_t subaddr_index_major, bool strict, uint64_t *blocks_to_unlock = NULL) const; + uint64_t unlocked_balance(uint32_t subaddr_index_major, bool strict, uint64_t *blocks_to_unlock = NULL, uint64_t *time_to_unlock = NULL) const; // locked & unlocked balance per subaddress of given or current subaddress account std::map<uint32_t, uint64_t> balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; + std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; // all locked & unlocked balances of all subaddress accounts uint64_t balance_all(bool strict) const; - uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL) const; + uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL, uint64_t *time_to_unlock = NULL) const; template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, @@ -1319,25 +1332,25 @@ private: crypto::public_key get_multisig_signing_public_key(const crypto::secret_key &skey) const; template<class t_request, class t_response> - inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET") + inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST") { if (m_offline) return false; boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - return epee::net_utils::invoke_http_json(uri, req, res, m_http_client, timeout, http_method); + return epee::net_utils::invoke_http_json(uri, req, res, *m_http_client, timeout, http_method); } template<class t_request, class t_response> - inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET") + inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST") { if (m_offline) return false; boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - return epee::net_utils::invoke_http_bin(uri, req, res, m_http_client, timeout, http_method); + return epee::net_utils::invoke_http_bin(uri, req, res, *m_http_client, timeout, http_method); } template<class t_request, class t_response> - inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0") + inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0") { if (m_offline) return false; boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id); + return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, *m_http_client, timeout, http_method, req_id); } bool set_ring_database(const std::string &filename); @@ -1403,11 +1416,18 @@ private: */ bool store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only = false); /*! - * \brief Load wallet information from wallet file. + * \brief Load wallet keys information from wallet file. * \param keys_file_name Name of wallet file * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); + /*! + * \brief Load wallet keys information from a string buffer. + * \param keys_buf Keys buffer to load + * \param password Password of keys buffer + */ + bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password); + bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt); void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); bool should_skip_block(const cryptonote::block &b, uint64_t height) const; void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); @@ -1487,7 +1507,7 @@ private: void on_device_button_request(uint64_t code); void on_device_button_pressed(); boost::optional<epee::wipeable_string> on_device_pin_request(); - boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); + boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device); void on_device_progress(const hw::device_progress& event); std::string get_rpc_status(const std::string &s) const; @@ -1502,7 +1522,7 @@ private: std::string m_wallet_file; std::string m_keys_file; std::string m_mms_file; - epee::net_utils::http::http_simple_client m_http_client; + const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client; hashchain m_blockchain; std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 41696d13b..4f5364269 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -85,7 +85,7 @@ bool wallet2::make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credit uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); epee::json_rpc::error error; - bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", "rpc_access_submit_nonce", req, res, error, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", "rpc_access_submit_nonce", req, res, error, *m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, error, res, "rpc_access_submit_nonce"); THROW_WALLET_EXCEPTION_IF(res.credits < pre_call_credits, error::wallet_internal_error, "RPC payment did not increase balance"); diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index db2e2344b..30eed07e7 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -428,10 +428,10 @@ namespace tools try { res.balance = req.all_accounts ? m_wallet->balance_all(req.strict) : m_wallet->balance(req.account_index, req.strict); - res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(req.strict, &res.blocks_to_unlock) : m_wallet->unlocked_balance(req.account_index, req.strict, &res.blocks_to_unlock); + res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(req.strict, &res.blocks_to_unlock, &res.time_to_unlock) : m_wallet->unlocked_balance(req.account_index, req.strict, &res.blocks_to_unlock, &res.time_to_unlock); res.multisig_import_needed = m_wallet->multisig() && m_wallet->has_multisig_partial_key_images(); std::map<uint32_t, std::map<uint32_t, uint64_t>> balance_per_subaddress_per_account; - std::map<uint32_t, std::map<uint32_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress_per_account; + std::map<uint32_t, std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>>> unlocked_balance_per_subaddress_per_account; if (req.all_accounts) { for (uint32_t account_index = 0; account_index < m_wallet->get_num_subaddress_accounts(); ++account_index) @@ -451,7 +451,7 @@ namespace tools { uint32_t account_index = p.first; std::map<uint32_t, uint64_t> balance_per_subaddress = p.second; - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = unlocked_balance_per_subaddress_per_account[account_index]; + std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress = unlocked_balance_per_subaddress_per_account[account_index]; std::set<uint32_t> address_indices; if (!req.all_accounts && !req.address_indices.empty()) { @@ -471,7 +471,8 @@ namespace tools info.address = m_wallet->get_subaddress_as_str(index); info.balance = balance_per_subaddress[i]; info.unlocked_balance = unlocked_balance_per_subaddress[i].first; - info.blocks_to_unlock = unlocked_balance_per_subaddress[i].second; + info.blocks_to_unlock = unlocked_balance_per_subaddress[i].second.first; + info.time_to_unlock = unlocked_balance_per_subaddress[i].second.second; info.label = m_wallet->get_subaddress_label(index); info.num_unspent_outputs = std::count_if(transfers.begin(), transfers.end(), [&](const tools::wallet2::transfer_details& td) { return !td.m_spent && td.m_subaddr_index == index; }); res.per_subaddress.emplace_back(std::move(info)); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index a212b79e6..507ff4f6c 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 17 +#define WALLET_RPC_VERSION_MINOR 18 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -84,6 +84,7 @@ namespace wallet_rpc std::string label; uint64_t num_unspent_outputs; uint64_t blocks_to_unlock; + uint64_t time_to_unlock; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) @@ -94,6 +95,7 @@ namespace wallet_rpc KV_SERIALIZE(label) KV_SERIALIZE(num_unspent_outputs) KV_SERIALIZE(blocks_to_unlock) + KV_SERIALIZE(time_to_unlock) END_KV_SERIALIZE_MAP() }; @@ -104,6 +106,7 @@ namespace wallet_rpc bool multisig_import_needed; std::vector<per_subaddress_info> per_subaddress; uint64_t blocks_to_unlock; + uint64_t time_to_unlock; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(balance) @@ -111,6 +114,7 @@ namespace wallet_rpc KV_SERIALIZE(multisig_import_needed) KV_SERIALIZE(per_subaddress) KV_SERIALIZE(blocks_to_unlock) + KV_SERIALIZE(time_to_unlock) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 5f2a3d077..42d14e91a 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -34,8 +34,8 @@ try: except: tests = DEFAULT_TESTS -N_MONERODS = 2 -N_WALLETS = 4 +N_MONERODS = 3 +N_WALLETS = 7 WALLET_DIRECTORY = builddir + "/functional-tests-directory" DIFFICULTY = 10 @@ -43,9 +43,17 @@ monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", st monerod_extra = [ [], ["--rpc-payment-address", "44SKxxLQw929wRF6BA9paQ1EWFshNnKhXM3qz6Mo3JGDE2YG3xyzVutMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUiqhcwR", "--rpc-payment-difficulty", str(DIFFICULTY), "--rpc-payment-credits", "5000", "--data-dir", builddir + "/functional-tests-directory/monerod1"], + ["--rpc-restricted-bind-port", "18482", "--data-dir", builddir + "/functional-tests-directory/monerod2"] ] -wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--daemon-port", "18180", "--log-level", "1"] +wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--log-level", "1"] wallet_extra = [ + ["--daemon-port", "18180"], + ["--daemon-port", "18180"], + ["--daemon-port", "18180"], + ["--daemon-port", "18180"], + ["--daemon-port", "18182"], + ["--daemon-port", "18182"], + ["--daemon-port", "18182"] ] command_lines = [] diff --git a/tests/functional_tests/make_test_signature.cc b/tests/functional_tests/make_test_signature.cc index 6ac1a6a86..f31816841 100644 --- a/tests/functional_tests/make_test_signature.cc +++ b/tests/functional_tests/make_test_signature.cc @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <stdio.h> +#include "misc_language.h" #include "string_tools.h" #include "rpc/rpc_payment_signature.h" @@ -69,6 +70,7 @@ int main(int argc, const char **argv) while (count--) { std::string signature = cryptonote::make_rpc_payment_signature(skey); + epee::misc_utils::sleep_no_w(1); printf("%s\n", signature.c_str()); } return 0; diff --git a/tests/functional_tests/rpc_payment.py b/tests/functional_tests/rpc_payment.py index 3bf995f0c..5f23c2022 100755 --- a/tests/functional_tests/rpc_payment.py +++ b/tests/functional_tests/rpc_payment.py @@ -59,12 +59,14 @@ class RPCPaymentTest(): return fields def refill_signatures(self): + self.signatures_time = time.time() + self.signatures = [] signatures = subprocess.check_output([self.make_test_signature, self.secret_key, '256']).decode('utf-8') for line in signatures.split(): self.signatures.append(line.rstrip()) def get_signature(self): - if len(self.signatures) == 0: + if len(self.signatures) == 0 or self.signatures_time + 10 < time.time(): self.refill_signatures() s = self.signatures[0] self.signatures = self.signatures[1:] diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py index c3d71aa9c..f7a39fa0c 100755 --- a/tests/functional_tests/transfer.py +++ b/tests/functional_tests/transfer.py @@ -55,7 +55,7 @@ class TransferTest(): def reset(self): print('Resetting blockchain') - daemon = Daemon() + daemon = Daemon(idx = 2) res = daemon.get_height() daemon.pop_blocks(res.height - 1) daemon.flush_txpool() @@ -69,7 +69,7 @@ class TransferTest(): ] self.wallet = [None] * len(seeds) for i in range(len(seeds)): - self.wallet[i] = Wallet(idx = i) + self.wallet[i] = Wallet(idx = i + 4) # close the wallet if any, will throw if none is loaded try: self.wallet[i].close_wallet() except: pass @@ -77,7 +77,7 @@ class TransferTest(): def mine(self): print("Mining some blocks") - daemon = Daemon() + daemon = Daemon(idx = 2) res = daemon.get_info() height = res.height @@ -89,7 +89,7 @@ class TransferTest(): assert res.height == height + 80 def transfer(self): - daemon = Daemon() + daemon = Daemon(idx = 2) print("Creating transfer to self") @@ -508,7 +508,7 @@ class TransferTest(): def check_get_bulk_payments(self): print('Checking get_bulk_payments') - daemon = Daemon() + daemon = Daemon(idx = 2) res = daemon.get_info() height = res.height @@ -544,7 +544,7 @@ class TransferTest(): def check_get_payments(self): print('Checking get_payments') - daemon = Daemon() + daemon = Daemon(idx = 2) res = daemon.get_info() height = res.height @@ -587,7 +587,8 @@ class TransferTest(): assert len(res.tx_blob_list) == 1 txes[i][1] = res.tx_blob_list[0] - daemon = Daemon() + daemon = Daemon(idx = 2) + restricted_daemon = Daemon(idx = 2, restricted_rpc = True) res = daemon.send_raw_transaction(txes[0][1]) assert res.not_relayed == False assert res.low_mixin == False @@ -598,6 +599,18 @@ class TransferTest(): assert res.overspend == False assert res.fee_too_low == False + res = restricted_daemon.send_raw_transaction(txes[0][1]) + assert res.not_relayed == False + assert res.low_mixin == False + assert res.double_spend == False + assert res.invalid_input == False + assert res.invalid_output == False + assert res.too_big == False + assert res.overspend == False + assert res.fee_too_low == False + + res = restricted_daemon.get_transactions([txes[0][0]]) + assert not 'txs' in res or len(res.txs) == 0 res = daemon.get_transactions([txes[0][0]]) assert len(res.txs) >= 1 tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0] @@ -615,6 +628,19 @@ class TransferTest(): assert res.fee_too_low == False assert res.too_few_outputs == False + res = restricted_daemon.send_raw_transaction(txes[1][1]) + assert res.not_relayed == False + assert res.low_mixin == False + assert res.double_spend == True + assert res.invalid_input == False + assert res.invalid_output == False + assert res.too_big == False + assert res.overspend == False + assert res.fee_too_low == False + assert res.too_few_outputs == False + + res = restricted_daemon.get_transactions([txes[0][0]]) + assert not 'txs' in res or len(res.txs) == 0 res = daemon.get_transactions([txes[0][0]]) assert len(res.txs) >= 1 tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0] @@ -623,13 +649,13 @@ class TransferTest(): def sweep_dust(self): print("Sweeping dust") - daemon = Daemon() + daemon = Daemon(idx = 2) self.wallet[0].refresh() res = self.wallet[0].sweep_dust() assert not 'tx_hash_list' in res or len(res.tx_hash_list) == 0 # there's just one, but it cannot meet the fee def sweep_single(self): - daemon = Daemon() + daemon = Daemon(idx = 2) print("Sending single output") @@ -685,7 +711,7 @@ class TransferTest(): assert len([t for t in res.transfers if t.key_image == ki]) == 1 def check_destinations(self): - daemon = Daemon() + daemon = Daemon(idx = 2) print("Checking transaction destinations") @@ -741,7 +767,7 @@ class TransferTest(): self.wallet[0].refresh() def check_tx_notes(self): - daemon = Daemon() + daemon = Daemon(idx = 2) print('Testing tx notes') res = self.wallet[0].get_transfers() @@ -758,7 +784,7 @@ class TransferTest(): assert res.notes == ['out txid', 'in txid'] def check_rescan(self): - daemon = Daemon() + daemon = Daemon(idx = 2) print('Testing rescan_spent') res = self.wallet[0].incoming_transfers(transfer_type = 'all') @@ -798,7 +824,7 @@ class TransferTest(): assert sorted(old_t_out, key = lambda k: k['txid']) == sorted(new_t_out, key = lambda k: k['txid']) def check_is_key_image_spent(self): - daemon = Daemon() + daemon = Daemon(idx = 2) print('Testing is_key_image_spent') res = self.wallet[0].incoming_transfers(transfer_type = 'all') diff --git a/tests/functional_tests/txpool.py b/tests/functional_tests/txpool.py index 27ae89764..2d7f69f3c 100755 --- a/tests/functional_tests/txpool.py +++ b/tests/functional_tests/txpool.py @@ -45,14 +45,14 @@ class TransferTest(): def reset(self): print('Resetting blockchain') - daemon = Daemon() + daemon = Daemon(idx=2) res = daemon.get_height() daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self): print('Creating wallet') - wallet = Wallet() + wallet = Wallet(idx = 4) # close the wallet if any, will throw if none is loaded try: wallet.close_wallet() except: pass @@ -61,8 +61,8 @@ class TransferTest(): def mine(self): print("Mining some blocks") - daemon = Daemon() - wallet = Wallet() + daemon = Daemon(idx = 2) + wallet = Wallet(idx = 4) daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80) wallet.refresh() @@ -70,8 +70,8 @@ class TransferTest(): def create_txes(self, address, ntxes): print('Creating ' + str(ntxes) + ' transactions') - daemon = Daemon() - wallet = Wallet() + daemon = Daemon(idx = 2) + wallet = Wallet(idx = 4) dst = {'address': address, 'amount': 1000000000000} @@ -83,8 +83,10 @@ class TransferTest(): return txes def check_empty_pool(self): - daemon = Daemon() + self.check_empty_rpc_pool(Daemon(idx = 2)) + self.check_empty_rpc_pool(Daemon(idx = 2, restricted_rpc = True)) + def check_empty_rpc_pool(self, daemon): res = daemon.get_transaction_pool_hashes() assert not 'tx_hashes' in res or len(res.tx_hashes) == 0 res = daemon.get_transaction_pool_stats() @@ -103,8 +105,9 @@ class TransferTest(): assert res.pool_stats.num_double_spends == 0 def check_txpool(self): - daemon = Daemon() - wallet = Wallet() + daemon = Daemon(idx = 2) + restricted_daemon = Daemon(idx = 2, restricted_rpc = True) + wallet = Wallet(idx = 4) res = daemon.get_info() height = res.height @@ -117,6 +120,7 @@ class TransferTest(): res = daemon.get_info() assert res.tx_pool_size == txpool_size + 5 txpool_size = res.tx_pool_size + self.check_empty_rpc_pool(restricted_daemon) res = daemon.get_transaction_pool() assert len(res.transactions) == txpool_size @@ -160,6 +164,7 @@ class TransferTest(): print('Flushing 2 transactions') txes_keys = list(txes.keys()) daemon.flush_txpool([txes_keys[1], txes_keys[3]]) + self.check_empty_rpc_pool(restricted_daemon) res = daemon.get_transaction_pool() assert len(res.transactions) == txpool_size - 2 assert len([x for x in res.transactions if x.id_hash == txes_keys[1]]) == 0 @@ -210,6 +215,7 @@ class TransferTest(): print('Flushing unknown transactions') unknown_txids = ['1'*64, '2'*64, '3'*64] daemon.flush_txpool(unknown_txids) + self.check_empty_rpc_pool(restricted_daemon) res = daemon.get_transaction_pool() assert len(res.transactions) == txpool_size - 2 diff --git a/tests/fuzz/http-client.cpp b/tests/fuzz/http-client.cpp index 1750838ae..ea6d5a2ad 100644 --- a/tests/fuzz/http-client.cpp +++ b/tests/fuzz/http-client.cpp @@ -29,6 +29,7 @@ #include "include_base_utils.h" #include "file_io_utils.h" #include "net/http_client.h" +#include "net/net_ssl.h" #include "fuzzer.h" class dummy_client @@ -46,6 +47,10 @@ public: data.clear(); return true; } + void set_ssl(epee::net_utils::ssl_options_t ssl_options) { } + bool is_connected(bool *ssl = NULL) { return true; } + uint64_t get_bytes_sent() const { return 1; } + uint64_t get_bytes_received() const { return 1; } void set_test_data(const std::string &s) { data = s; } diff --git a/tests/performance_tests/crypto_ops.h b/tests/performance_tests/crypto_ops.h index 5b215cef4..2859873ef 100644 --- a/tests/performance_tests/crypto_ops.h +++ b/tests/performance_tests/crypto_ops.h @@ -48,6 +48,7 @@ enum test_op op_scalarmultKey, op_scalarmultH, op_scalarmult8, + op_scalarmult8_p3, op_ge_dsm_precomp, op_ge_double_scalarmult_base_vartime, op_ge_double_scalarmult_precomp_vartime, @@ -105,6 +106,7 @@ public: case op_scalarmultKey: rct::scalarmultKey(point0, scalar0); break; case op_scalarmultH: rct::scalarmultH(scalar0); break; case op_scalarmult8: rct::scalarmult8(point0); break; + case op_scalarmult8_p3: rct::scalarmult8(p3_0,point0); break; case op_ge_dsm_precomp: ge_dsm_precomp(dsmp, &p3_0); break; case op_ge_double_scalarmult_base_vartime: ge_double_scalarmult_base_vartime(&tmp_p2, scalar0.bytes, &p3_0, scalar1.bytes); break; case op_ge_double_scalarmult_precomp_vartime: ge_double_scalarmult_precomp_vartime(&tmp_p2, scalar0.bytes, &p3_0, scalar1.bytes, precomp0); break; diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index bd7414c59..d3ba57270 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -254,6 +254,7 @@ int main(int argc, char** argv) TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmultKey); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmultH); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmult8); + TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmult8_p3); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_dsm_precomp); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_base_vartime); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_precomp_vartime); diff --git a/tests/trezor/daemon.cpp b/tests/trezor/daemon.cpp index 41af93f3f..aba835ae2 100644 --- a/tests/trezor/daemon.cpp +++ b/tests/trezor/daemon.cpp @@ -129,7 +129,7 @@ void mock_daemon::init() m_rpc_server.nettype(m_network_type); CHECK_AND_ASSERT_THROW_MES(m_protocol.init(m_vm), "Failed to initialize cryptonote protocol."); - CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port), "Failed to initialize RPC server."); + CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port, false), "Failed to initialize RPC server."); if (m_start_p2p) CHECK_AND_ASSERT_THROW_MES(m_server.init(m_vm), "Failed to initialize p2p server."); @@ -313,7 +313,7 @@ void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_addres { bool blocks_mined = false; const uint64_t start_height = get_height(); - const auto mining_timeout = std::chrono::seconds(30); + const auto mining_timeout = std::chrono::seconds(120); MDEBUG("Current height before mining: " << start_height); start_mining(miner_address); diff --git a/tests/trezor/daemon.h b/tests/trezor/daemon.h index 046b09a5d..4b8094e05 100644 --- a/tests/trezor/daemon.h +++ b/tests/trezor/daemon.h @@ -76,7 +76,7 @@ public: typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; typedef nodetool::node_server<t_protocol_raw> t_node_server; - static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(60); + static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(120); cryptonote::core * m_core; t_protocol_raw m_protocol; diff --git a/tests/trezor/trezor_tests.cpp b/tests/trezor/trezor_tests.cpp index a867a4047..f5867f5e7 100644 --- a/tests/trezor/trezor_tests.cpp +++ b/tests/trezor/trezor_tests.cpp @@ -38,6 +38,7 @@ using namespace cryptonote; #include <boost/regex.hpp> +#include <common/apply_permutation.h> #include "common/util.h" #include "common/command_line.h" #include "trezor_tests.h" @@ -72,9 +73,10 @@ namespace #define TREZOR_SETUP_CHAIN(NAME) do { \ ++tests_count; \ try { \ - setup_chain(core, trezor_base, chain_path, fix_chain, vm_core); \ + setup_chain(core, trezor_base, chain_path, fix_chain, vm_core); \ } catch (const std::exception& ex) { \ - failed_tests.emplace_back("gen_trezor_base " #NAME); \ + MERROR("Chain setup failed for " << NAME); \ + throw; \ } \ } while(0) @@ -136,10 +138,11 @@ int main(int argc, char* argv[]) hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device()); // shim device for call tracking // Bootstrapping common chain & accounts - const uint8_t initial_hf = (uint8_t)get_env_long("TEST_MIN_HF", 11); - const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", 11); + const uint8_t initial_hf = (uint8_t)get_env_long("TEST_MIN_HF", 12); + const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", 12); + auto sync_test = get_env_long("TEST_KI_SYNC", 1); MINFO("Test versions " << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); - MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "]"); + MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "], sync-test: " << sync_test); cryptonote::core core_obj(nullptr); cryptonote::core * const core = &core_obj; @@ -181,7 +184,7 @@ int main(int argc, char* argv[]) trezor_base.daemon(daemon); // Hard-fork independent tests - if (hf == initial_hf) + if (hf == initial_hf && sync_test > 0) { TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_without_refresh, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_live_refresh, core, trezor_base); @@ -191,7 +194,6 @@ int main(int argc, char* argv[]) TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base); @@ -338,10 +340,7 @@ static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, if (!unserialize_chain_from_file(events, trezor_base, chain_path)) { MERROR("Failed to deserialize data from file: " << chain_path); - if (!fix_chain) - { - throw std::runtime_error("Chain load error"); - } + CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain load error"); } else { trezor_base.load(events); @@ -648,6 +647,8 @@ void gen_trezor_base::fork(gen_trezor_base & other) other.m_alice_account = m_alice_account; other.m_eve_account = m_eve_account; other.m_trezor = m_trezor; + other.m_generator.set_events(&other.m_events); + other.m_generator.set_network_type(m_network_type); } void gen_trezor_base::clear() @@ -700,6 +701,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) // Events, custom genesis so it matches wallet genesis auto & generator = m_generator; // macro shortcut + generator.set_events(&events); + generator.set_network_type(m_network_type); cryptonote::block blk_gen; std::vector<size_t> block_weights; @@ -852,6 +855,8 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events) { init_fields(); m_events = events; + m_generator.set_events(&m_events); + m_generator.set_network_type(m_network_type); unsigned acc_idx = 0; cryptonote::account_base * accounts[] = {TREZOR_ACCOUNT_ORDERING}; @@ -919,29 +924,19 @@ void gen_trezor_base::rewind_blocks(std::vector<test_event_entry>& events, size_ void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events) { // If current test requires higher hard-fork, move it up - const auto current_hf = m_hard_forks.back().first; - - if (current_hf > m_top_hard_fork) - { - throw std::runtime_error("Generated chain hardfork is higher than desired maximum"); - } - - if (m_rct_config.bp_version == 2 && m_top_hard_fork < 10) - { - throw std::runtime_error("Desired maximum is too low for BPv2"); - } + auto current_hf = m_hard_forks.back().first; + CHECK_AND_ASSERT_THROW_MES(current_hf <= m_top_hard_fork, "Generated chain hardfork is higher than desired maximum"); + CHECK_AND_ASSERT_THROW_MES(m_rct_config.bp_version != 2 || m_top_hard_fork >= 10, "Desired maximum is too low for BPv2"); - if (current_hf < m_top_hard_fork) + for(;current_hf < m_top_hard_fork; current_hf+=1) { + auto const hf_to_add = current_hf + 1; auto hardfork_height = num_blocks(events); - ADD_HARDFORK(m_hard_forks, m_top_hard_fork, hardfork_height); - add_top_hfork(events, m_hard_forks); - MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)m_top_hard_fork); - if (current_hf < 10) - { // buffer blocks, add 10 to apply v10 rules - rewind_blocks(events, 10, m_top_hard_fork); - } + ADD_HARDFORK(m_hard_forks, hf_to_add, hardfork_height); + add_top_hfork(events, m_hard_forks); + MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)hf_to_add); + rewind_blocks(events, 10, hf_to_add); } } @@ -1271,7 +1266,6 @@ void gen_trezor_base::set_hard_fork(uint8_t hf) #define TREZOR_SKIP_IF_VERSION_LEQ(x) if (m_trezor->get_version() <= x) { MDEBUG("Test skipped"); return true; } #define TREZOR_TEST_PAYMENT_ID "\xde\xad\xc0\xde\xde\xad\xc0\xde" -#define TREZOR_TEST_PAYMENT_ID_LONG "\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde" tsx_builder * tsx_builder::sources(std::vector<cryptonote::tx_source_entry> & sources, std::vector<size_t> & selected_transfers) { @@ -1424,13 +1418,26 @@ tsx_builder * tsx_builder::construct_pending_tx(tools::wallet2::pending_tx &ptx, std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = m_destinations; + auto sources_copy = m_sources; auto change_addr = m_from->get_account().get_keys().m_account_address; bool r = construct_tx_and_get_tx_key(m_from->get_account().get_keys(), subaddresses, m_sources, destinations_copy, change_addr, extra ? extra.get() : std::vector<uint8_t>(), tx, 0, tx_key, additional_tx_keys, true, m_rct_config, nullptr); - CHECK_AND_ASSERT_THROW_MES(r, "Transaction construction failed"); + // Selected transfers permutation + std::vector<size_t> ins_order; + for (size_t n = 0; n < m_sources.size(); ++n) + { + for (size_t idx = 0; idx < sources_copy.size(); ++idx) + { + CHECK_AND_ASSERT_THROW_MES((size_t)sources_copy[idx].real_output < sources_copy[idx].outputs.size(), "Invalid real_output"); + if (sources_copy[idx].outputs[sources_copy[idx].real_output].second.dest == m_sources[n].outputs[m_sources[n].real_output].second.dest) + ins_order.push_back(idx); + } + } + CHECK_AND_ASSERT_THROW_MES(ins_order.size() == m_sources.size(), "Failed to work out sources permutation"); + ptx.key_images = ""; ptx.fee = TESTS_DEFAULT_FEE; ptx.dust = 0; @@ -1438,6 +1445,7 @@ tsx_builder * tsx_builder::construct_pending_tx(tools::wallet2::pending_tx &ptx, ptx.tx = tx; ptx.change_dts = m_destinations.back(); ptx.selected_transfers = m_selected_transfers; + tools::apply_permutation(ins_order, ptx.selected_transfers); ptx.tx_key = tx_key; ptx.additional_tx_keys = additional_tx_keys; ptx.dests = m_destinations; @@ -1671,22 +1679,6 @@ bool gen_trezor_1utxo_paymentid_short_integrated::generate(std::vector<test_even TREZOR_TEST_SUFFIX(); } -bool gen_trezor_1utxo_paymentid_long::generate(std::vector<test_event_entry>& events) -{ - TREZOR_TEST_PREFIX(); - t_builder->cur_height(num_blocks(events) - 1) - ->mixin(TREZOR_TEST_MIXIN) - ->fee(TREZOR_TEST_FEE) - ->from(m_wl_alice.get(), 0) - ->compute_sources(boost::none, MK_COINS(1), -1, -1) - ->add_destination(m_eve_account, false, 1000) - ->payment_id(TREZOR_TEST_PAYMENT_ID_LONG) - ->rct_config(m_rct_config) - ->build_tx(); - - TREZOR_TEST_SUFFIX(); -} - bool gen_trezor_4utxo::generate(std::vector<test_event_entry>& events) { TREZOR_TEST_PREFIX(); diff --git a/tests/trezor/trezor_tests.h b/tests/trezor/trezor_tests.h index 46eb5e6a5..67714f102 100644 --- a/tests/trezor/trezor_tests.h +++ b/tests/trezor/trezor_tests.h @@ -264,12 +264,6 @@ public: bool generate(std::vector<test_event_entry>& events) override; }; -class gen_trezor_1utxo_paymentid_long : public gen_trezor_base -{ -public: - bool generate(std::vector<test_event_entry>& events) override; -}; - class gen_trezor_4utxo : public gen_trezor_base { public: diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 42bbb26bb..2e93f9e67 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2019, The Monero Project +// Copyright (c) 2014-2020, The Monero Project // // All rights reserved. // @@ -45,6 +45,7 @@ #include "boost/archive/portable_binary_iarchive.hpp" #include "boost/archive/portable_binary_oarchive.hpp" #include "byte_slice.h" +#include "byte_stream.h" #include "crypto/crypto.h" #include "hex.h" #include "net/net_utils_base.h" @@ -809,6 +810,259 @@ TEST(ByteSlice, GetSlice) EXPECT_TRUE(boost::range::equal(base_string, original)); } +TEST(ByteStream, Construction) +{ + EXPECT_TRUE(std::is_default_constructible<epee::byte_stream>()); + EXPECT_TRUE(std::is_move_constructible<epee::byte_stream>()); + EXPECT_FALSE(std::is_copy_constructible<epee::byte_stream>()); + EXPECT_TRUE(std::is_move_assignable<epee::byte_stream>()); + EXPECT_FALSE(std::is_copy_assignable<epee::byte_stream>()); +} + +TEST(ByteStream, Noexcept) +{ + EXPECT_TRUE(std::is_nothrow_default_constructible<epee::byte_stream>()); + EXPECT_TRUE(std::is_nothrow_move_constructible<epee::byte_stream>()); + EXPECT_TRUE(std::is_nothrow_move_assignable<epee::byte_stream>()); + + epee::byte_stream lvalue; + const epee::byte_stream clvalue; + + EXPECT_TRUE(noexcept(lvalue.data())); + EXPECT_TRUE(noexcept(clvalue.data())); + EXPECT_TRUE(noexcept(lvalue.tellp())); + EXPECT_TRUE(noexcept(clvalue.tellp())); + EXPECT_TRUE(noexcept(lvalue.available())); + EXPECT_TRUE(noexcept(clvalue.available())); + EXPECT_TRUE(noexcept(lvalue.size())); + EXPECT_TRUE(noexcept(clvalue.size())); + EXPECT_TRUE(noexcept(lvalue.capacity())); + EXPECT_TRUE(noexcept(clvalue.capacity())); + EXPECT_TRUE(noexcept(lvalue.put_unsafe(4))); + EXPECT_TRUE(noexcept(lvalue.take_buffer())); +} + +TEST(ByteStream, Empty) +{ + epee::byte_stream stream; + + EXPECT_EQ(epee::byte_stream::default_increase(), stream.increase_size()); + + EXPECT_EQ(nullptr, stream.data()); + EXPECT_EQ(nullptr, stream.tellp()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(0u, stream.capacity()); + + const auto buf = stream.take_buffer(); + EXPECT_EQ(nullptr, buf.get()); + EXPECT_EQ(nullptr, stream.data()); + EXPECT_EQ(nullptr, stream.tellp()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(0u, stream.capacity()); +} + +TEST(ByteStream, Write) +{ + using boost::range::equal; + using byte_span = epee::span<const std::uint8_t>; + + static constexpr const std::uint8_t source[] = + {0xde, 0xad, 0xbe, 0xef, 0xef}; + + std::vector<std::uint8_t> bytes; + epee::byte_stream stream{4}; + + EXPECT_EQ(4u, stream.increase_size()); + + stream.write({source, 3}); + bytes.insert(bytes.end(), source, source + 3); + EXPECT_EQ(3u, stream.size()); + EXPECT_EQ(1u, stream.available()); + EXPECT_EQ(4u, stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); + + stream.write({source, 2}); + bytes.insert(bytes.end(), source, source + 2); + EXPECT_EQ(5u, stream.size()); + EXPECT_EQ(3u, stream.available()); + EXPECT_EQ(8u, stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); + + stream.write({source, 5}); + bytes.insert(bytes.end(), source, source + 5); + EXPECT_EQ(10u, stream.size()); + EXPECT_EQ(2u, stream.available()); + EXPECT_EQ(12u, stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); + + stream.write({source, 2}); + bytes.insert(bytes.end(), source, source + 2); + EXPECT_EQ(12u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(12u, stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); + + stream.write({source, 5}); + bytes.insert(bytes.end(), source, source + 5); + EXPECT_EQ(17u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(17u, stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); +} + +TEST(ByteStream, Put) +{ + using boost::range::equal; + using byte_span = epee::span<const std::uint8_t>; + + std::vector<std::uint8_t> bytes; + epee::byte_stream stream; + + for (std::uint8_t i = 0; i < 200; ++i) + { + bytes.push_back(i); + stream.put(i); + } + + EXPECT_EQ(200u, stream.size()); + EXPECT_EQ(epee::byte_stream::default_increase() - 200, stream.available()); + EXPECT_EQ(epee::byte_stream::default_increase(), stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); +} + +TEST(ByteStream, Reserve) +{ + using boost::range::equal; + using byte_span = epee::span<const std::uint8_t>; + + static constexpr const std::uint8_t source[] = + {0xde, 0xad, 0xbe, 0xef, 0xef}; + + std::vector<std::uint8_t> bytes; + epee::byte_stream stream{4}; + + EXPECT_EQ(4u, stream.increase_size()); + + stream.reserve(100); + EXPECT_EQ(100u, stream.capacity()); + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(100u, stream.available()); + + for (std::size_t i = 0; i < 100 / sizeof(source); ++i) + { + stream.write(source); + bytes.insert(bytes.end(), source, source + sizeof(source)); + } + + EXPECT_EQ(100u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(100u, stream.capacity()); + EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()})); +} + +TEST(ByteStream, TakeBuffer) +{ + using boost::range::equal; + using byte_span = epee::span<const std::uint8_t>; + + static constexpr const std::uint8_t source[] = + {0xde, 0xad, 0xbe, 0xef, 0xef}; + + epee::byte_stream stream; + + stream.write(source); + ASSERT_EQ(sizeof(source), stream.size()); + EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()})); + + const auto buffer = stream.take_buffer(); + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(0u, stream.capacity()); + EXPECT_EQ(nullptr, stream.data()); + EXPECT_EQ(nullptr, stream.tellp()); + EXPECT_TRUE(equal(source, byte_span{buffer.get(), sizeof(source)})); +} + +TEST(ByteStream, Move) +{ + using boost::range::equal; + using byte_span = epee::span<const std::uint8_t>; + + static constexpr const std::uint8_t source[] = + {0xde, 0xad, 0xbe, 0xef, 0xef}; + + epee::byte_stream stream{10}; + stream.write(source); + + epee::byte_stream stream2{std::move(stream)}; + + EXPECT_EQ(10u, stream.increase_size()); + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(0u, stream.capacity()); + EXPECT_EQ(nullptr, stream.data()); + EXPECT_EQ(nullptr, stream.tellp()); + + EXPECT_EQ(10u, stream2.increase_size()); + EXPECT_EQ(5u, stream2.size()); + EXPECT_EQ(5u, stream2.available()); + EXPECT_EQ(10u, stream2.capacity()); + EXPECT_NE(nullptr, stream2.data()); + EXPECT_NE(nullptr, stream2.tellp()); + EXPECT_TRUE(equal(source, byte_span{stream2.data(), stream2.size()})); + + stream = epee::byte_stream{}; + + EXPECT_EQ(epee::byte_stream::default_increase(), stream.increase_size()); + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(0u, stream.capacity()); + EXPECT_EQ(nullptr, stream.data()); + EXPECT_EQ(nullptr, stream.tellp()); + + stream = std::move(stream2); + + EXPECT_EQ(10u, stream.increase_size()); + EXPECT_EQ(5u, stream.size()); + EXPECT_EQ(5u, stream.available()); + EXPECT_EQ(10u, stream.capacity()); + EXPECT_NE(nullptr, stream.data()); + EXPECT_NE(nullptr, stream.tellp()); + EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()})); + + EXPECT_EQ(10u, stream2.increase_size()); + EXPECT_EQ(0u, stream2.size()); + EXPECT_EQ(0u, stream2.available()); + EXPECT_EQ(0u, stream2.capacity()); + EXPECT_EQ(nullptr, stream2.data()); + EXPECT_EQ(nullptr, stream2.tellp()); +} + +TEST(ByteStream, ToByteSlice) +{ + using boost::range::equal; + using byte_span = epee::span<const std::uint8_t>; + + static constexpr const std::uint8_t source[] = + {0xde, 0xad, 0xbe, 0xef, 0xef}; + + epee::byte_stream stream; + + stream.write(source); + EXPECT_EQ(sizeof(source), stream.size()); + EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()})); + + const epee::byte_slice slice{std::move(stream)}; + EXPECT_EQ(0u, stream.size()); + EXPECT_EQ(0u, stream.available()); + EXPECT_EQ(0u, stream.capacity()); + EXPECT_EQ(nullptr, stream.data()); + EXPECT_EQ(nullptr, stream.tellp()); + EXPECT_TRUE(equal(source, slice)); +} + TEST(ToHex, String) { EXPECT_TRUE(epee::to_hex::string(nullptr).empty()); @@ -841,6 +1095,9 @@ TEST(HexLocale, String) // decoding it this way also, ignoring spaces and colons between the numbers hex.assign("00:ff 0f:f0"); EXPECT_EQ(source, epee::from_hex_locale::to_vector(hex)); + + hex.append("f0"); + EXPECT_EQ(source, epee::from_hex_locale::to_vector(boost::string_ref{hex.data(), hex.size() - 2})); } TEST(ToHex, Array) diff --git a/tests/unit_tests/json_serialization.cpp b/tests/unit_tests/json_serialization.cpp index 6f98d854d..5873d0ab6 100644 --- a/tests/unit_tests/json_serialization.cpp +++ b/tests/unit_tests/json_serialization.cpp @@ -3,10 +3,10 @@ #include <boost/range/adaptor/indexed.hpp> #include <gtest/gtest.h> #include <rapidjson/document.h> -#include <rapidjson/stringbuffer.h> #include <rapidjson/writer.h> #include <vector> +#include "byte_stream.h" #include "crypto/hash.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/cryptonote_basic.h" @@ -86,14 +86,14 @@ namespace template<typename T> T test_json(const T& value) { - rapidjson::StringBuffer buffer; + epee::byte_stream buffer; { - rapidjson::Writer<rapidjson::StringBuffer> dest{buffer}; + rapidjson::Writer<epee::byte_stream> dest{buffer}; cryptonote::json::toJsonValue(dest, value); } rapidjson::Document doc; - doc.Parse(buffer.GetString()); + doc.Parse(reinterpret_cast<const char*>(buffer.data()), buffer.size()); if (doc.HasParseError() || !doc.IsObject()) { throw cryptonote::json::PARSE_FAIL(); diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index 62b101a42..d2ef920ef 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -583,7 +583,9 @@ TEST_F(levin_notify, stem_without_padding) { const std::size_t sent = context->process_send_queue(); if (sent && is_stem) + { EXPECT_EQ(1u, (context - contexts_.begin()) % 2); + } send_count += sent; } @@ -653,7 +655,9 @@ TEST_F(levin_notify, local_without_padding) { const std::size_t sent = context->process_send_queue(); if (sent && is_stem) + { EXPECT_EQ(1u, (context - contexts_.begin()) % 2); + } send_count += sent; } diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index 36cb28ae0..f5aef4796 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -1702,6 +1702,45 @@ TEST(zmq, read_write) EXPECT_EQ(message, *received); } +TEST(zmq, read_write_slice) +{ + net::zmq::context context{zmq_init(1)}; + ASSERT_NE(nullptr, context); + + net::zmq::socket send_socket{zmq_socket(context.get(), ZMQ_REQ)}; + net::zmq::socket recv_socket{zmq_socket(context.get(), ZMQ_REP)}; + ASSERT_NE(nullptr, send_socket); + ASSERT_NE(nullptr, recv_socket); + + ASSERT_EQ(0u, zmq_bind(recv_socket.get(), "inproc://testing")); + ASSERT_EQ(0u, zmq_connect(send_socket.get(), "inproc://testing")); + + std::string message; + message.resize(1024); + crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0]))); + + { + epee::byte_slice slice_message{{epee::strspan<std::uint8_t>(message)}}; + ASSERT_TRUE(bool(net::zmq::send(std::move(slice_message), send_socket.get()))); + EXPECT_TRUE(slice_message.empty()); + } + + const expect<std::string> received = net::zmq::receive(recv_socket.get()); + ASSERT_TRUE(bool(received)); + EXPECT_EQ(message, *received); +} + +TEST(zmq, write_slice_fail) +{ + std::string message; + message.resize(1024); + crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0]))); + + epee::byte_slice slice_message{std::move(message)}; + EXPECT_FALSE(bool(net::zmq::send(std::move(slice_message), nullptr))); + EXPECT_TRUE(slice_message.empty()); +} + TEST(zmq, read_write_multipart) { net::zmq::context context{zmq_init(1)}; diff --git a/tests/unit_tests/ringct.cpp b/tests/unit_tests/ringct.cpp index 075aeac40..64fcbba4c 100644 --- a/tests/unit_tests/ringct.cpp +++ b/tests/unit_tests/ringct.cpp @@ -1077,8 +1077,16 @@ TEST(ringct, H) TEST(ringct, mul8) { + ge_p3 p3; + rct::key key; ASSERT_EQ(rct::scalarmult8(rct::identity()), rct::identity()); + rct::scalarmult8(p3,rct::identity()); + ge_p3_tobytes(key.bytes, &p3); + ASSERT_EQ(key, rct::identity()); ASSERT_EQ(rct::scalarmult8(rct::H), rct::scalarmultKey(rct::H, rct::EIGHT)); + rct::scalarmult8(p3,rct::H); + ge_p3_tobytes(key.bytes, &p3); + ASSERT_EQ(key, rct::scalarmultKey(rct::H, rct::EIGHT)); ASSERT_EQ(rct::scalarmultKey(rct::scalarmultKey(rct::H, rct::INV_EIGHT), rct::EIGHT), rct::H); } diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 749d9ed88..074f8de37 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -32,10 +32,11 @@ from .rpc import JSONRPC class Daemon(object): - def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0): + def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0, restricted_rpc = False): + base = 18480 if restricted_rpc else 18180 self.host = host self.port = port - self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else 18180+idx)) + self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else base+idx)) def getblocktemplate(self, address, prev_block = "", client = ""): getblocktemplate = { |