diff options
102 files changed, 5333 insertions, 876 deletions
diff --git a/.travis.yml b/.travis.yml index 07b92aaff..516e4f8e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ before_script: - if [ -n "$OSX_SDK" -a -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-g++ \$(which $HOST-g++-posix)"; fi - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-gcc \$(which $HOST-gcc-posix)"; fi - - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "CONFIG_SHELL= make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi + - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi script: - git submodule init && git submodule update - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` diff --git a/ANONYMITY_NETWORKS.md b/ANONYMITY_NETWORKS.md index feb8528da..6eede44aa 100644 --- a/ANONYMITY_NETWORKS.md +++ b/ANONYMITY_NETWORKS.md @@ -160,25 +160,6 @@ the system clock is noticeably off (and therefore more fingerprintable), linking the public IPv4/IPv6 connections with the anonymity networks will be more difficult. -### Bandwidth Usage - -An ISP can passively monitor `monerod` connections from a node and observe when -a transaction is sent over a Tor/I2P connection via timing analysis + size of -data sent during that timeframe. I2P should provide better protection against -this attack - its connections are not circuit based. However, if a node is -only using I2P for broadcasting Monero transactions, the total aggregate of -I2P data would also leak information. - -#### Mitigation - -There is no current mitigation for the user right now. This attack is fairly -sophisticated, and likely requires support from the internet host of a Monero -user. - -In the near future, "whitening" the amount of data sent over anonymity network -connections will be performed. An attempt will be made to make a transaction -broadcast indistinguishable from a peer timed sync command. - ### Intermittent Monero Syncing If a user only runs `monerod` to send a transaction then quit, this can also @@ -208,3 +189,36 @@ is a tradeoff in potential isses. Also, anyone attempting this strategy really wants to uncover a user, it seems unlikely that this would be performed against every Tor/I2P user. +### I2P/Tor Stream Used Twice + +If a single I2P/Tor stream is used 2+ times for transmitting a transaction, the +operator of the hidden service can conclude that both transactions came from the +same source. If the subsequent transactions spend a change output from the +earlier transactions, this will also reveal the "real" spend in the ring +signature. This issue was (primarily) raised by @secparam on Twitter. + +#### Mitigation + +`monerod` currently selects two outgoing connections every 5 minutes for +transmitting transactions over I2P/Tor. Using outgoing connections prevents an +adversary from making many incoming connections to obtain information (this +technique was taken from Dandelion). Outgoing connections also do not have a +persistent public key identity - the creation of a new circuit will generate +a new public key identity. The lock time on a change address is ~20 minutes, so +`monerod` will have rotated its selected outgoing connections several times in +most cases. However, the number of outgoing connections is typically a small +fixed number, so there is a decent probability of re-use with the same public +key identity. + +@secparam (twitter) recommended changing circuits (Tor) as an additional +precaution. This is likely not a good idea - forcibly requesting Tor to change +circuits is observable by the ISP. Instead, `monerod` should likely disconnect +from peers ocassionally. Tor will rotate circuits every ~10 minutes, so +establishing new connections will use a new public key identity and make it +more difficult for the hidden service to link information. This process will +have to be done carefully because closing/reconnecting connections can also +leak information to hidden services if done improperly. + +At the current time, if users need to frequently make transactions, I2P/Tor +will improve privacy from ISPs and other common adversaries, but still have +some metadata leakages to unknown hidden service operators. diff --git a/Dockerfile b/Dockerfile index 999290c95..a6fab3823 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,9 +29,9 @@ ENV CFLAGS='-fPIC' ENV CXXFLAGS='-fPIC' #Cmake -ARG CMAKE_VERSION=3.14.0 +ARG CMAKE_VERSION=3.14.6 ARG CMAKE_VERSION_DOT=v3.14 -ARG CMAKE_HASH=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 +ARG CMAKE_HASH=4e8ea11cabe459308671b476469eace1622e770317a15951d7b55a82ccaaccb9 RUN set -ex \ && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \ && echo "${CMAKE_HASH} cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \ @@ -42,9 +42,9 @@ RUN set -ex \ && make install ## Boost -ARG BOOST_VERSION=1_69_0 -ARG BOOST_VERSION_DOT=1.69.0 -ARG BOOST_HASH=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 +ARG BOOST_VERSION=1_70_0 +ARG BOOST_VERSION_DOT=1.70.0 +ARG BOOST_HASH=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 RUN set -ex \ && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ @@ -69,8 +69,8 @@ RUN set -ex \ ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION} # ZMQ -ARG ZMQ_VERSION=v4.3.1 -ARG ZMQ_HASH=2cb1240db64ce1ea299e00474c646a2453a8435b +ARG ZMQ_VERSION=v4.3.2 +ARG ZMQ_HASH=a84ffa12b2eb3569ced199660bac5ad128bff1f0 RUN set -ex \ && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ && cd libzmq \ @@ -82,8 +82,8 @@ RUN set -ex \ && ldconfig # zmq.hpp -ARG CPPZMQ_VERSION=v4.3.0 -ARG CPPZMQ_HASH=213da0b04ae3b4d846c9abc46bab87f86bfb9cf4 +ARG CPPZMQ_VERSION=v4.4.1 +ARG CPPZMQ_HASH=f5b36e563598d48fcc0d82e589d3596afef945ae RUN set -ex \ && git clone https://github.com/zeromq/cppzmq.git -b ${CPPZMQ_VERSION} \ && cd cppzmq \ @@ -103,8 +103,8 @@ RUN set -ex \ && make install # Sodium -ARG SODIUM_VERSION=1.0.17 -ARG SODIUM_HASH=b732443c442239c2e0184820e9b23cca0de0828c +ARG SODIUM_VERSION=1.0.18 +ARG SODIUM_HASH=4f5e89fa84ce1d178a6765b8b46f2b6f91216677 RUN set -ex \ && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ && cd libsodium \ @@ -116,8 +116,8 @@ RUN set -ex \ && make install # Udev -ARG UDEV_VERSION=v3.2.7 -ARG UDEV_HASH=4758e346a14126fc3a964de5831e411c27ebe487 +ARG UDEV_VERSION=v3.2.8 +ARG UDEV_HASH=d69f3f28348123ab7fa0ebac63ec2fd16800c5e0 RUN set -ex \ && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ && cd eudev \ @@ -152,8 +152,8 @@ RUN set -ex \ && make install # Protobuf -ARG PROTOBUF_VERSION=v3.7.0 -ARG PROTOBUF_HASH=582743bf40c5d3639a70f98f183914a2c0cd0680 +ARG PROTOBUF_VERSION=v3.7.1 +ARG PROTOBUF_HASH=6973c3a5041636c1d8dc5f7f6c8c1f3c15bc63d6 RUN set -ex \ && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ && cd protobuf \ diff --git a/LEVIN_PROTOCOL.md b/LEVIN_PROTOCOL.md new file mode 100644 index 000000000..207509146 --- /dev/null +++ b/LEVIN_PROTOCOL.md @@ -0,0 +1,165 @@ +# Levin Protocol +This is a document explaining the current design of the levin protocol, as +used by Monero. The protocol is largely inherited from cryptonote, but has +undergone some changes. + +This document also may differ from the `struct bucket_head2` in Monero's +code slightly - the spec here is slightly more strict to allow for +extensibility. + +One of the goals of this document is to clearly indicate what is being sent +"on the wire" to identify metadata that could de-anonymize users over I2P/Tor. +These issues will be addressed as they are found. See `ANONMITY_NETWORKS.md` in +the top-level folder for any outstanding issues. + +> This document does not currently list all data being sent by the monero +> protocol, that portion is a work-in-progress. Please take the time to do it +> if interested in learning about Monero p2p traffic! + + +## Header +This header is sent for every Monero p2p message. + +``` + 0 1 2 3 + 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| 0x01 | 0x21 | 0x01 | 0x01 | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| 0x01 | 0x01 | 0x01 | 0x01 | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length | +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| E. Response | Command ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Return Code ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |Q|S|B|E| Reserved ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | 0x01 | 0x00 | 0x00 | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| 0x00 | ++-+-+-+-+-+-+-+-+ +``` + +### Signature +The first 8 bytes are the "signature" which helps identify the protocol (in +case someone connected to the wrong port, etc). The comments indicate that byte +sequence is from "benders nightmare". + +This also can be used by deep packet inspection (DPI) engines to identify +Monero when the link is not encrypted. SSL has been proposed as a means to +mitigate this issue, but BIP-151 or the Noise protocol should also be considered. + +### Length +The length is an unsigned 64-bit little endian integer. The length does _not_ +include the header. + +The implementation currently rejects received messages that exceed 100 MB +(base 10) by default. + +### Expect Response +A zero-byte if no response is expected from the peer, and a non-zero byte if a +response is expected from the peer. Peers must respond to requests with this +flag in the same order that they were received, however, other messages can be +sent between responses. + +There are some commands in the +[cryptonote protocol](#cryptonote-protocol-commands) where a response is +expected from the peer, but this flag is not set. Those responses are returned +as notify messages and can be sent in any order by the peer. + +### Command +An unsigned 32-bit little endian integer representing the Monero specific +command being invoked. + +### Return Code +A signed 32-bit little integer integer representing the response from the peer +from the last command that was invoked. This is `0` for request messages. + +### Flags + * `Q` - Bit is set if the message is a request. + * `S` - Bit is set if the message is a response. + * `B` - Bit is set if this is a the beginning of a [fragmented message](#fragmented-messages). + * `E` - Bit is set if this is the end of a [fragmented message](#fragmented-messages). + +### Version +A fixed value of `1` as an unsigned 32-bit little endian integer. + + +## Message Flow +The protocol can be subdivided into: (1) notifications, (2) requests, +(3) responses, (4) fragmented messages, and (5) dummy messages. Response +messages must be sent in the same order that a peer issued a request message. +A peer does not have to send a response immediately following a request - any +other message type can be sent instead. + +### Notifications +Notifications are one-way messages that can be sent at any time without +an expectation of a response from the peer. The `Q` bit must be set, the `S`, +`B` and `E` bits must be unset, and the `Expect Response` field must be zeroed. + +Some notifications must be in response to other notifications. This is not +part of the levin messaging layer, and is described in the +[commands](#commands) section. + +### Requests +Requests are the basis of the admin protocol for Monero. The `Q` bit must be +set, the `S`, `B` and `E` bits must be unset, and the `Expect Response` field +must be non-zero. The peer is expected to send a response message with the same +`command` number. + +### Responses +Response message can only be sent after a peer first issues a request message. +Responses must have the `S` bit set, the `Q`, `B` and `E` bits unset, and have +a zeroed `Expect Response` field. The `Command` field must be the same value +that was sent in the request message. The `Return Code` is specific to the +`Command` being issued (see [commands])(#commands)). + +### Fragmented +Fragmented messages were introduced for the "white noise" feature for i2p/tor. +A transaction can be sent in fragments to conceal when "real" data is being +sent instead of dummy messages. Only one fragmented message can be sent at a +time, and bits `B` and `E` are never set at the same time +(see [dummy messages](#dummy)). The re-constructed message must contain a +levin header for a different (non-fragment) message type. + +The `Q` and `S` bits are never set and the `Expect Response` field must always +be zero. The first fragment has the `B` bit set, neither `B` nor `E` is set for +"middle" fragments, and `E` is set for the last fragment. + +### Dummy +Dummy messages have the `B` and `E` bits set, the `Q` and `S` bits unset, and +the `Expect Reponse` field zeroed. When a message of this type is received, the +contents can be safely ignored. + + +## Commands +### P2P (Admin) Commands + +#### (`1001` Request) Handshake +#### (`1001` Response) Handshake +#### (`1002` Request) Timed Sync +#### (`1002` Response) Timed Sync +#### (`1003` Request) Ping +#### (`1003` Response) Ping +#### (`1004` Request) Stat Info +#### (`1004` Response) Stat Info +#### (`1005` Request) Network State +#### (`1005` Response) Network State +#### (`1006` Request) Peer ID +#### (`1006` Reponse) Peer ID +#### (`1007` Request) Support Flags +#### (`1007` Response) Support Flags + +### Cryptonote Protocol Commands + +#### (`2001` Notification) New Block +#### (`2002` Notification) New Transactions +#### (`2003` Notification) Request Get Objects +#### (`2004` Notification) Response Get Objects +#### (`2006` Notification) Request Chain +#### (`2007` Notification) Response Chain Entry +#### (`2008` Notification) New Fluffy Block +#### (`2009` Notification) Request Fluffy Missing TX @@ -550,13 +550,12 @@ The produced binaries still link libc dynamically. If the binary is compiled on Packages are available for -* Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. +* Debian Bullseye and Sid ```bash - snap install monero --beta + sudo apt install monero ``` - -Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. +More info and versions in the [Debian package tracker](https://tracker.debian.org/pkg/monero). * Arch Linux (via [AUR](https://aur.archlinux.org/)): - Stable release: [`monero`](https://aur.archlinux.org/packages/monero) diff --git a/cmake/GenVersion.cmake b/cmake/GenVersion.cmake index b2ccfbc34..992539507 100644 --- a/cmake/GenVersion.cmake +++ b/cmake/GenVersion.cmake @@ -36,6 +36,7 @@ if(RET) message(WARNING "Cannot determine current commit. Make sure that you are building either from a Git working tree or from a source archive.") set(VERSIONTAG "unknown") + set(VERSION_IS_RELEASE "false") configure_file("src/version.cpp.in" "${TO}") else() string(SUBSTRING ${COMMIT} 0 9 COMMIT) @@ -47,6 +48,7 @@ else() if(NOT TAGGEDCOMMIT) message(WARNING "Cannot determine most recent tag. Make sure that you are building either from a Git working tree or from a source archive.") set(VERSIONTAG "${COMMIT}") + set(VERSION_IS_RELEASE "false") else() message(STATUS "The most recent tag was at ${TAGGEDCOMMIT}") @@ -54,9 +56,11 @@ else() if(COMMIT STREQUAL TAGGEDCOMMIT) message(STATUS "You are building a tagged release") set(VERSIONTAG "release") + set(VERSION_IS_RELEASE "true") else() message(STATUS "You are ahead of or behind a tagged release") set(VERSIONTAG "${COMMIT}") + set(VERSION_IS_RELEASE "false") endif() endif() diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h new file mode 100644 index 000000000..1fbba101e --- /dev/null +++ b/contrib/epee/include/byte_slice.h @@ -0,0 +1,145 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <string> +#include <vector> + +#include "span.h" + +namespace epee +{ + struct byte_slice_data; + + struct release_byte_slice + { + void operator()(byte_slice_data*) const noexcept; + }; + + /*! Inspired by slices in golang. Storage is thread-safe reference counted, + allowing for cheap copies or range selection on the bytes. The bytes + owned by this class are always immutable. + + The functions `operator=`, `take_slice` and `remove_prefix` may alter the + reference count for the backing store, which will invalidate pointers + previously returned if the reference count is zero. Be careful about + "caching" pointers in these circumstances. */ + class byte_slice + { + /* A custom reference count is used instead of shared_ptr because it allows + for an allocation optimization for the span constructor. This also + reduces the size of this class by one pointer. */ + std::unique_ptr<byte_slice_data, release_byte_slice> storage_; + span<const std::uint8_t> portion_; // within storage_ + + //! Internal use only; use to increase `storage` reference count. + byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept; + + struct adapt_buffer{}; + + template<typename T> + explicit byte_slice(const adapt_buffer, T&& buffer); + + public: + using value_type = std::uint8_t; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using pointer = const std::uint8_t*; + using const_pointer = const std::uint8_t*; + using reference = std::uint8_t; + using const_reference = std::uint8_t; + using iterator = pointer; + using const_iterator = const_pointer; + + //! Construct empty slice. + byte_slice() noexcept + : storage_(nullptr), portion_() + {} + + //! Construct empty slice + byte_slice(std::nullptr_t) noexcept + : byte_slice() + {} + + //! Scatter-gather (copy) multiple `sources` into a single allocated slice. + explicit byte_slice(std::initializer_list<span<const std::uint8_t>> sources); + + //! Convert `buffer` into a slice using one allocation for shared count. + explicit byte_slice(std::vector<std::uint8_t>&& buffer); + + //! Convert `buffer` into a slice using one allocation for shared count. + explicit byte_slice(std::string&& buffer); + + byte_slice(byte_slice&& source) noexcept; + ~byte_slice() noexcept = default; + + //! \note May invalidate previously retrieved pointers. + byte_slice& operator=(byte_slice&&) noexcept; + + //! \return A shallow (cheap) copy of the data from `this` slice. + byte_slice clone() const noexcept { return {storage_.get(), portion_}; } + + iterator begin() const noexcept { return portion_.begin(); } + const_iterator cbegin() const noexcept { return portion_.begin(); } + + iterator end() const noexcept { return portion_.end(); } + const_iterator cend() const noexcept { return portion_.end(); } + + bool empty() const noexcept { return storage_ == nullptr; } + const std::uint8_t* data() const noexcept { return portion_.data(); } + std::size_t size() const noexcept { return portion_.size(); } + + /*! Drop bytes from the beginning of `this` slice. + + \note May invalidate previously retrieved pointers. + \post `this->size() = this->size() - std::min(this->size(), max_bytes)` + \post `if (this->size() <= max_bytes) this->data() = nullptr` + \return Number of bytes removed. */ + std::size_t remove_prefix(std::size_t max_bytes) noexcept; + + /*! "Take" bytes from the beginning of `this` slice. + + \note May invalidate previously retrieved pointers. + \post `this->size() = this->size() - std::min(this->size(), max_bytes)` + \post `if (this->size() <= max_bytes) this->data() = nullptr` + \return Slice containing the bytes removed from `this` slice. */ + byte_slice take_slice(std::size_t max_bytes) noexcept; + + /*! Return a shallow (cheap) copy of a slice from `begin` and `end` offsets. + + \throw std::out_of_range If `end < begin`. + \throw std::out_of_range If `size() < end`. + \return Slice starting at `data() + begin` of size `end - begin`. */ + byte_slice get_slice(std::size_t begin, std::size_t end) const; + }; +} // epee + diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index e07e16d91..13747b0c8 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -45,6 +45,9 @@ #include "readline_buffer.h" #endif +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "console_handler" + namespace epee { class async_stdin_reader @@ -96,7 +99,7 @@ namespace epee res = true; } - if (!eos()) + if (!eos() && m_read_status != state_cancelled) m_read_status = state_init; return res; @@ -122,6 +125,14 @@ namespace epee } } + void cancel() + { + boost::unique_lock<boost::mutex> lock(m_response_mutex); + m_read_status = state_cancelled; + m_has_read_request = false; + m_response_cv.notify_one(); + } + private: bool start_read() { @@ -162,6 +173,9 @@ namespace epee while (m_run.load(std::memory_order_relaxed)) { + if (m_read_status == state_cancelled) + return false; + fd_set read_set; FD_ZERO(&read_set); FD_SET(stdin_fileno, &read_set); @@ -179,6 +193,9 @@ namespace epee #else while (m_run.load(std::memory_order_relaxed)) { + if (m_read_status == state_cancelled) + return false; + int retval = ::WaitForSingleObject(::GetStdHandle(STD_INPUT_HANDLE), 100); switch (retval) { @@ -219,7 +236,8 @@ reread: case rdln::full: break; } #else - std::getline(std::cin, line); + if (m_read_status != state_cancelled) + std::getline(std::cin, line); #endif read_ok = !std::cin.eof() && !std::cin.fail(); } @@ -303,7 +321,7 @@ eof: template<class chain_handler> bool run(chain_handler ch_handler, std::function<std::string(void)> prompt, const std::string& usage = "", std::function<void(void)> exit_handler = NULL) { - return run(prompt, usage, [&](const std::string& cmd) { return ch_handler(cmd); }, exit_handler); + return run(prompt, usage, [&](const boost::optional<std::string>& cmd) { return ch_handler(cmd); }, exit_handler); } void stop() @@ -312,6 +330,12 @@ eof: m_stdin_reader.stop(); } + void cancel() + { + m_cancel = true; + m_stdin_reader.cancel(); + } + void print_prompt() { std::string prompt = m_prompt(); @@ -360,18 +384,23 @@ eof: std::cout << std::endl; break; } + + if (m_cancel) + { + MDEBUG("Input cancelled"); + cmd_handler(boost::none); + m_cancel = false; + continue; + } if (!get_line_ret) { MERROR("Failed to read line."); } + string_tools::trim(command); LOG_PRINT_L2("Read command: " << command); - if (command.empty()) - { - continue; - } - else if(cmd_handler(command)) + if(cmd_handler(command)) { continue; } @@ -401,6 +430,7 @@ eof: private: async_stdin_reader m_stdin_reader; std::atomic<bool> m_running = {true}; + std::atomic<bool> m_cancel = {false}; std::function<std::string(void)> m_prompt; }; @@ -482,8 +512,16 @@ eof: class command_handler { public: typedef boost::function<bool (const std::vector<std::string> &)> callback; + typedef boost::function<bool (void)> empty_callback; typedef std::map<std::string, std::pair<callback, std::pair<std::string, std::string>>> lookup; + command_handler(): + m_unknown_command_handler([](const std::vector<std::string>&){return false;}), + m_empty_command_handler([](){return true;}), + m_cancel_handler([](){return true;}) + { + } + std::string get_usage() { std::stringstream ss; @@ -516,25 +554,45 @@ eof: #endif } + void set_unknown_command_handler(const callback& hndlr) + { + m_unknown_command_handler = hndlr; + } + + void set_empty_command_handler(const empty_callback& hndlr) + { + m_empty_command_handler = hndlr; + } + + void set_cancel_handler(const empty_callback& hndlr) + { + m_cancel_handler = hndlr; + } + bool process_command_vec(const std::vector<std::string>& cmd) { - if(!cmd.size()) - return false; + if(!cmd.size() || (cmd.size() == 1 && !cmd[0].size())) + return m_empty_command_handler(); auto it = m_command_handlers.find(cmd.front()); if(it == m_command_handlers.end()) - return false; + return m_unknown_command_handler(cmd); std::vector<std::string> cmd_local(cmd.begin()+1, cmd.end()); return it->second.first(cmd_local); } - bool process_command_str(const std::string& cmd) + bool process_command_str(const boost::optional<std::string>& cmd) { + if (!cmd) + return m_cancel_handler(); std::vector<std::string> cmd_v; - boost::split(cmd_v,cmd,boost::is_any_of(" "), boost::token_compress_on); + boost::split(cmd_v,*cmd,boost::is_any_of(" "), boost::token_compress_on); return process_command_vec(cmd_v); } private: lookup m_command_handlers; + callback m_unknown_command_handler; + empty_callback m_empty_command_handler; + empty_callback m_cancel_handler; }; /************************************************************************/ @@ -572,6 +630,11 @@ eof: { m_console_handler.print_prompt(); } + + void cancel_input() + { + m_console_handler.cancel(); + } }; ///* work around because of broken boost bind */ diff --git a/contrib/epee/include/int-util.h b/contrib/epee/include/int-util.h index 0ed6505ff..8ef5be40a 100644 --- a/contrib/epee/include/int-util.h +++ b/contrib/epee/include/int-util.h @@ -129,9 +129,12 @@ static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uin return remainder; } +#define IDENT16(x) ((uint16_t) (x)) #define IDENT32(x) ((uint32_t) (x)) #define IDENT64(x) ((uint64_t) (x)) +#define SWAP16(x) ((((uint16_t) (x) & 0x00ff) << 8) | \ + (((uint16_t) (x) & 0xff00) >> 8)) #define SWAP32(x) ((((uint32_t) (x) & 0x000000ff) << 24) | \ (((uint32_t) (x) & 0x0000ff00) << 8) | \ (((uint32_t) (x) & 0x00ff0000) >> 8) | \ @@ -145,10 +148,18 @@ static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uin (((uint64_t) (x) & 0x00ff000000000000) >> 40) | \ (((uint64_t) (x) & 0xff00000000000000) >> 56)) +static inline uint16_t ident16(uint16_t x) { return x; } static inline uint32_t ident32(uint32_t x) { return x; } static inline uint64_t ident64(uint64_t x) { return x; } #ifndef __OpenBSD__ +# if defined(__ANDROID__) && defined(__swap16) && !defined(swap16) +# define swap16 __swap16 +# elif !defined(swap16) +static inline uint16_t swap16(uint16_t x) { + return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8); +} +# endif # if defined(__ANDROID__) && defined(__swap32) && !defined(swap32) # define swap32 __swap32 # elif !defined(swap32) @@ -176,6 +187,12 @@ static inline uint64_t swap64(uint64_t x) { static inline void mem_inplace_ident(void *mem UNUSED, size_t n UNUSED) { } #undef UNUSED +static inline void mem_inplace_swap16(void *mem, size_t n) { + size_t i; + for (i = 0; i < n; i++) { + ((uint16_t *) mem)[i] = swap16(((const uint16_t *) mem)[i]); + } +} static inline void mem_inplace_swap32(void *mem, size_t n) { size_t i; for (i = 0; i < n; i++) { @@ -189,6 +206,9 @@ static inline void mem_inplace_swap64(void *mem, size_t n) { } } +static inline void memcpy_ident16(void *dst, const void *src, size_t n) { + memcpy(dst, src, 2 * n); +} static inline void memcpy_ident32(void *dst, const void *src, size_t n) { memcpy(dst, src, 4 * n); } @@ -196,6 +216,12 @@ static inline void memcpy_ident64(void *dst, const void *src, size_t n) { memcpy(dst, src, 8 * n); } +static inline void memcpy_swap16(void *dst, const void *src, size_t n) { + size_t i; + for (i = 0; i < n; i++) { + ((uint16_t *) dst)[i] = swap16(((const uint16_t *) src)[i]); + } +} static inline void memcpy_swap32(void *dst, const void *src, size_t n) { size_t i; for (i = 0; i < n; i++) { @@ -220,6 +246,14 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e #endif #if BYTE_ORDER == LITTLE_ENDIAN +#define SWAP16LE IDENT16 +#define SWAP16BE SWAP16 +#define swap16le ident16 +#define swap16be swap16 +#define mem_inplace_swap16le mem_inplace_ident +#define mem_inplace_swap16be mem_inplace_swap16 +#define memcpy_swap16le memcpy_ident16 +#define memcpy_swap16be memcpy_swap16 #define SWAP32LE IDENT32 #define SWAP32BE SWAP32 #define swap32le ident32 @@ -239,6 +273,14 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e #endif #if BYTE_ORDER == BIG_ENDIAN +#define SWAP16BE IDENT16 +#define SWAP16LE SWAP16 +#define swap16be ident16 +#define swap16le swap16 +#define mem_inplace_swap16be mem_inplace_ident +#define mem_inplace_swap16le mem_inplace_swap16 +#define memcpy_swap16be memcpy_ident16 +#define memcpy_swap16le memcpy_swap16 #define SWAP32BE IDENT32 #define SWAP32LE SWAP32 #define swap32be ident32 diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index b38ab5399..3a2c5341d 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -49,10 +49,12 @@ #include <boost/asio/ssl.hpp> #include <boost/array.hpp> #include <boost/noncopyable.hpp> -#include <boost/shared_ptr.hpp> +#include <boost/shared_ptr.hpp> //! \TODO Convert to std::shared_ptr #include <boost/enable_shared_from_this.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/thread/thread.hpp> +#include <memory> +#include "byte_slice.h" #include "net_utils_base.h" #include "syncobj.h" #include "connection_basic.hpp" @@ -90,25 +92,24 @@ namespace net_utils public: typedef typename t_protocol_handler::connection_context t_connection_context; - struct shared_state : connection_basic_shared_state + struct shared_state : connection_basic_shared_state, t_protocol_handler::config_type { shared_state() - : connection_basic_shared_state(), pfilter(nullptr), config(), stop_signal_sent(false) + : connection_basic_shared_state(), t_protocol_handler::config_type(), pfilter(nullptr), stop_signal_sent(false) {} i_connection_filter* pfilter; - typename t_protocol_handler::config_type config; bool stop_signal_sent; }; /// Construct a connection with the given io_service. explicit connection( boost::asio::io_service& io_service, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, epee::net_utils::ssl_support_t ssl_support); explicit connection( boost::asio::ip::tcp::socket&& sock, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, epee::net_utils::ssl_support_t ssl_support); @@ -135,8 +136,7 @@ namespace net_utils private: //----------------- i_service_endpoint --------------------- - virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) - virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data + virtual bool do_send(byte_slice message); ///< (see do_send from i_service_endpoint) virtual bool send_done(); virtual bool close(); virtual bool call_run_once_service_io(); @@ -145,6 +145,8 @@ namespace net_utils virtual bool add_ref(); virtual bool release(); //------------------------------------------------------ + bool do_send_chunk(byte_slice chunk); ///< will send (or queue) a part of data. internal use only + boost::shared_ptr<connection<t_protocol_handler> > safe_shared_from_this(); bool shutdown(); /// Handle completion of a receive operation. @@ -269,7 +271,13 @@ namespace net_utils typename t_protocol_handler::config_type& get_config_object() { assert(m_state != nullptr); // always set in constructor - return m_state->config; + return *m_state; + } + + std::shared_ptr<typename t_protocol_handler::config_type> get_config_shared() + { + assert(m_state != nullptr); // always set in constructor + return {m_state}; } int get_binded_port(){return m_port;} @@ -350,7 +358,7 @@ namespace net_utils bool is_thread_worker(); - const boost::shared_ptr<typename connection<t_protocol_handler>::shared_state> m_state; + const std::shared_ptr<typename connection<t_protocol_handler>::shared_state> m_state; /// The io_service used to perform asynchronous operations. struct worker diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 19e9c9af9..12a87071a 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -68,7 +68,7 @@ namespace epee namespace net_utils { template<typename T> - T& check_and_get(boost::shared_ptr<T>& ptr) + T& check_and_get(std::shared_ptr<T>& ptr) { CHECK_AND_ASSERT_THROW_MES(bool(ptr), "shared_state cannot be null"); return *ptr; @@ -81,7 +81,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> connection<t_protocol_handler>::connection( boost::asio::io_service& io_service, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, ssl_support_t ssl_support ) @@ -91,13 +91,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> connection<t_protocol_handler>::connection( boost::asio::ip::tcp::socket&& sock, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, ssl_support_t ssl_support ) : connection_basic(std::move(sock), state, ssl_support), - m_protocol_handler(this, check_and_get(state).config, context), + m_protocol_handler(this, check_and_get(state), context), buffer_ssl_init_fill(0), m_connection_type( connection_type ), m_throttle_speed_in("speed_in", "throttle_speed_in"), @@ -378,7 +378,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(!recv_res) { //_info("[sock " << socket().native_handle() << "] protocol_want_close"); - //some error in protocol, protocol handler ask to close connection boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); bool do_shutdown = false; @@ -520,7 +519,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) { + bool connection<t_protocol_handler>::do_send(byte_slice message) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -529,6 +528,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (m_was_shutdown) return false; // TODO avoid copy + std::uint8_t const* const message_data = message.data(); + const std::size_t message_size = message.size(); + const double factor = 32; // TODO config typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) ); @@ -538,13 +540,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) CHECK_AND_ASSERT_MES(! (chunksize_max<0), false, "Negative chunksize_max" ); // make sure it is unsigned before removin sign with cast: long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ; - if (allow_split && (cb > chunksize_max_unsigned)) { + if (allow_split && (message_size > chunksize_max_unsigned)) { { // LOCK: chunking epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** - MDEBUG("do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr); - t_safe all = cb; // all bytes to send - t_safe pos = 0; // current sending position + MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<message_data); // 01234567890 // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 @@ -554,40 +554,25 @@ PRAGMA_WARNING_DISABLE_VS(4355) // char* buf = new char[ bufsize ]; bool all_ok = true; - while (pos < all) { - t_safe lenall = all-pos; // length from here to end - t_safe len = std::min( chunksize_good , lenall); // take a smaller part - CHECK_AND_ASSERT_MES(len<=chunksize_good, false, "len too large"); - // pos=8; len=4; all=10; len=3; - - CHECK_AND_ASSERT_MES(! (len<0), false, "negative len"); // check before we cast away sign: - unsigned long long int len_unsigned = static_cast<long long int>( len ); - CHECK_AND_ASSERT_MES(len>0, false, "len not strictly positive"); // (redundant) - CHECK_AND_ASSERT_MES(len_unsigned < std::numeric_limits<size_t>::max(), false, "Invalid len_unsigned"); // yeap we want strong < then max size, to be sure - - void *chunk_start = ((char*)ptr) + pos; - MDEBUG("chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos); - CHECK_AND_ASSERT_MES(chunk_start >= ptr, false, "Pointer wraparound"); // not wrapped around address? - //std::memcpy( (void*)buf, chunk_start, len); - - MDEBUG("part of " << lenall << ": pos="<<pos << " len="<<len); - - bool ok = do_send_chunk(chunk_start, len); // <====== *** + while (!message.empty()) { + byte_slice chunk = message.take_slice(chunksize_good); + + MDEBUG("chunk_start="<<chunk.data()<<" ptr="<<message_data<<" pos="<<(chunk.data() - message_data)); + MDEBUG("part of " << message.size() << ": pos="<<(chunk.data() - message_data) << " len="<<chunk.size()); + + bool ok = do_send_chunk(std::move(chunk)); // <====== *** all_ok = all_ok && ok; if (!all_ok) { - MDEBUG("do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr); + MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<message_data); MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless " - << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb); + << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << message_size); return false; // partial failure in sending } - pos = pos+len; - CHECK_AND_ASSERT_MES(pos >0, false, "pos <= 0"); - // (in catch block, or uniq pointer) delete buf; } // each chunk - MDEBUG("do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr); + MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<message_data); MDEBUG("do_send() m_connection_type = " << m_connection_type); @@ -595,7 +580,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } // LOCK: chunking } // a big block (to be chunked) - all chunks else { // small block - return do_send_chunk(ptr,cb); // just send as 1 big chunk + return do_send_chunk(std::move(message)); // just send as 1 big chunk } CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); @@ -603,7 +588,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb) + bool connection<t_protocol_handler>::do_send_chunk(byte_slice chunk) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -615,7 +600,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) double current_speed_up; { CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); - m_throttle_speed_out.handle_trafic_exact(cb); + m_throttle_speed_out.handle_trafic_exact(chunk.size()); current_speed_up = m_throttle_speed_out.get_current_speed(); } context.m_current_speed_up = current_speed_up; @@ -623,7 +608,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //_info("[sock " << socket().native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); - context.m_send_cnt += cb; + context.m_send_cnt += chunk.size(); //some data should be wrote to stream //request complete @@ -644,7 +629,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) }*/ long int ms = 250 + (rand()%50); - MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep + MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<chunk.size()); // XXX debug sleep m_send_que_lock.unlock(); boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); m_send_que_lock.lock(); @@ -657,12 +642,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) } } - m_send_que.resize(m_send_que.size()+1); - m_send_que.back().assign((const char*)ptr, cb); - + m_send_que.push_back(std::move(chunk)); + if(m_send_que.size() > 1) { // active operation should be in progress, nothing to do, just wait last operation callback - auto size_now = cb; + auto size_now = m_send_que.back().size(); MDEBUG("do_send_chunk() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size()); //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function @@ -680,7 +664,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) auto size_now = m_send_que.front().size(); MDEBUG("do_send_chunk() NOW SENSD: packet="<<size_now<<" B"); if (speed_limit_is_enabled()) - do_send_handler_write( ptr , size_now ); // (((H))) + do_send_handler_write( m_send_que.back().data(), m_send_que.back().size() ); // (((H))) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size"); reset_timer(get_default_timeout(), false); @@ -908,7 +892,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> boosted_tcp_server<t_protocol_handler>::boosted_tcp_server( t_connection_type connection_type ) : - m_state(boost::make_shared<typename connection<t_protocol_handler>::shared_state>()), + m_state(std::make_shared<typename connection<t_protocol_handler>::shared_state>()), m_io_service_local_instance(new worker()), io_service_(m_io_service_local_instance->io_service), acceptor_(io_service_), @@ -927,7 +911,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_connection_type connection_type) : - m_state(boost::make_shared<typename connection<t_protocol_handler>::shared_state>()), + m_state(std::make_shared<typename connection<t_protocol_handler>::shared_state>()), io_service_(extarnal_io_service), acceptor_(io_service_), acceptor_ipv6(io_service_), diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp index 2acc6cdda..2f60f7604 100644 --- a/contrib/epee/include/net/connection_basic.hpp +++ b/contrib/epee/include/net/connection_basic.hpp @@ -49,6 +49,7 @@ #include <boost/asio.hpp> #include <boost/asio/ssl.hpp> +#include "byte_slice.h" #include "net/net_utils_base.h" #include "net/net_ssl.h" #include "syncobj.h" @@ -99,7 +100,7 @@ class connection_basic_pimpl; // PIMPL for this class class connection_basic { // not-templated base class for rapid developmet of some code parts // beware of removing const, net_utils::connection is sketchily doing a cast to prevent storing ptr twice - const boost::shared_ptr<connection_basic_shared_state> m_state; + const std::shared_ptr<connection_basic_shared_state> m_state; public: std::unique_ptr< connection_basic_pimpl > mI; // my Implementation @@ -108,7 +109,7 @@ class connection_basic { // not-templated base class for rapid developmet of som volatile uint32_t m_want_close_connection; std::atomic<bool> m_was_shutdown; critical_section m_send_que_lock; - std::list<std::string> m_send_que; + std::deque<byte_slice> m_send_que; volatile bool m_is_multithreaded; /// Strand to ensure the connection's handlers are not called concurrently. boost::asio::io_service::strand strand_; @@ -118,8 +119,8 @@ class connection_basic { // not-templated base class for rapid developmet of som public: // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator - connection_basic(boost::asio::ip::tcp::socket&& socket, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); - connection_basic(boost::asio::io_service &io_service, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); + connection_basic(boost::asio::ip::tcp::socket&& socket, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); + connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); virtual ~connection_basic() noexcept(false); diff --git a/contrib/epee/include/net/enums.h b/contrib/epee/include/net/enums.h index 078a4b274..2f27d07f9 100644 --- a/contrib/epee/include/net/enums.h +++ b/contrib/epee/include/net/enums.h @@ -49,7 +49,7 @@ namespace net_utils { invalid = 0, public_ = 1, // public is keyword - i2p = 2, + i2p = 2, // order from here changes priority of selection for origin TXes tor = 3 }; diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index 790d0f3b1..19bdf4ff0 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -591,11 +591,12 @@ namespace net_utils std::string response_data = get_response_header(response); //LOG_PRINT_L0("HTTP_SEND: << \r\n" << response_data + response.m_body); - LOG_PRINT_L3("HTTP_RESPONSE_HEAD: << \r\n" << response_data); - - m_psnd_hndlr->do_send((void*)response_data.data(), response_data.size()); + LOG_PRINT_L3("HTTP_RESPONSE_HEAD: << \r\n" << response_data); + if ((response.m_body.size() && (query_info.m_http_method != http::http_method_head)) || (query_info.m_http_method == http::http_method_options)) - m_psnd_hndlr->do_send((void*)response.m_body.data(), response.m_body.size()); + response_data += response.m_body; + + m_psnd_hndlr->do_send(byte_slice{std::move(response_data)}); m_psnd_hndlr->send_done(); return res; } diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index a88a1eb49..f9b6f9a81 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -29,7 +29,11 @@ #ifndef _LEVIN_BASE_H_ #define _LEVIN_BASE_H_ +#include <cstdint> + +#include "byte_slice.h" #include "net_utils_base.h" +#include "span.h" #define LEVIN_SIGNATURE 0x0101010101012101LL //Bender's nightmare @@ -72,6 +76,8 @@ namespace levin #define LEVIN_PACKET_REQUEST 0x00000001 #define LEVIN_PACKET_RESPONSE 0x00000002 +#define LEVIN_PACKET_BEGIN 0x00000004 +#define LEVIN_PACKET_END 0x00000008 #define LEVIN_PROTOCOL_VER_0 0 @@ -118,9 +124,30 @@ namespace levin } } + //! \return Intialized levin header. + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; + + //! \return A levin notification message. + byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); + + /*! Generate a dummy levin message. + \param noise_bytes Total size of the returned `byte_slice`. + \return `nullptr` if `noise_size` is smaller than the levin header. + Otherwise, a dummy levin message. */ + byte_slice make_noise_notify(std::size_t noise_bytes); + + /*! Generate 1+ levin messages that are identical to the noise message size. + + \param noise Each levin message will be identical to the size of this + message. The bytes from this message will be used for padding. + \return `nullptr` if `noise.size()` is less than the levin header size. + Otherwise, a levin notification message OR 2+ levin fragment messages. + Each message is `noise.size()` in length. */ + byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); } } #endif //_LEVIN_BASE_H_ + diff --git a/contrib/epee/include/net/levin_protocol_handler.h b/contrib/epee/include/net/levin_protocol_handler.h index 791766762..c510cfd79 100644 --- a/contrib/epee/include/net/levin_protocol_handler.h +++ b/contrib/epee/include/net/levin_protocol_handler.h @@ -157,10 +157,9 @@ namespace levin m_current_head.m_return_code = m_config.m_pcommands_handler->invoke(m_current_head.m_command, buff_to_invoke, return_buff, m_conn_context); m_current_head.m_cb = return_buff.size(); m_current_head.m_have_to_return_data = false; - std::string send_buff((const char*)&m_current_head, sizeof(m_current_head)); - send_buff += return_buff; - if(!m_psnd_hndlr->do_send(send_buff.data(), send_buff.size())) + return_buff.insert(0, (const char*)&m_current_head, sizeof(m_current_head)); + if(!m_psnd_hndlr->do_send(byte_slice{std::move(return_buff)})) return false; } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 8d7ffb2c2..41f01e9a0 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -32,6 +32,7 @@ #include <boost/smart_ptr/make_shared.hpp> #include <atomic> +#include <deque> #include "levin_base.h" #include "buffer.h" @@ -91,6 +92,7 @@ public: int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); + int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); bool request_callback(boost::uuids::uuid connection_id); @@ -117,6 +119,22 @@ public: template<class t_connection_context = net_utils::connection_context_base> class async_protocol_handler { + std::string m_fragment_buffer; + + bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + { + const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); + if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + return false; + + MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb + << ", flags" << head.m_flags + << ", r?=" << head.m_have_to_return_data + <<", cmd = " << head.m_command + << ", ver=" << head.m_protocol_version); + return true; + } + public: typedef t_connection_context connection_context; typedef async_protocol_handler_config<t_connection_context> config_type; @@ -136,7 +154,6 @@ public: critical_section m_local_inv_buff_lock; std::string m_local_inv_buff; - critical_section m_send_lock; critical_section m_call_lock; volatile uint32_t m_wait_count; @@ -376,7 +393,12 @@ public: return false; } - if(m_cache_in_buffer.size() + cb > m_config.m_max_packet_size) + // these should never fail, but do runtime check for safety + CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()"); + CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()"); + + // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public + if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size()) { MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size << ", packet received " << m_cache_in_buffer.size() + cb @@ -408,8 +430,38 @@ public: } break; } + { + std::string temp{}; epee::span<const uint8_t> buff_to_invoke = m_cache_in_buffer.carve((std::string::size_type)m_current_head.m_cb); + m_state = stream_state_head; + + // abstract_tcp_server2.h manages max bandwidth for a p2p link + if (!(m_current_head.m_flags & (LEVIN_PACKET_REQUEST | LEVIN_PACKET_RESPONSE))) + { + // special noise/fragment command + static constexpr const uint32_t both_flags = (LEVIN_PACKET_BEGIN | LEVIN_PACKET_END); + if ((m_current_head.m_flags & both_flags) == both_flags) + break; // noise message, skip to next message + + if (m_current_head.m_flags & LEVIN_PACKET_BEGIN) + m_fragment_buffer.clear(); + + m_fragment_buffer.append(reinterpret_cast<const char*>(buff_to_invoke.data()), buff_to_invoke.size()); + if (!(m_current_head.m_flags & LEVIN_PACKET_END)) + break; // skip to next message + + if (m_fragment_buffer.size() < sizeof(bucket_head2)) + { + MERROR(m_connection_context << "Fragmented data too small for levin header"); + return false; + } + + temp = std::move(m_fragment_buffer); + m_fragment_buffer.clear(); + std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2)); + buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)}; + } bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE); @@ -458,43 +510,33 @@ public: if(m_current_head.m_have_to_return_data) { std::string return_buff; - m_current_head.m_return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, - buff_to_invoke, - return_buff, - m_connection_context); - m_current_head.m_cb = return_buff.size(); - m_current_head.m_have_to_return_data = false; - m_current_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; - m_current_head.m_flags = LEVIN_PACKET_RESPONSE; -#if BYTE_ORDER == LITTLE_ENDIAN - std::string send_buff((const char*)&m_current_head, sizeof(m_current_head)); -#else - bucket_head2 head = m_current_head; - head.m_signature = SWAP64LE(head.m_signature); - head.m_cb = SWAP64LE(head.m_cb); - head.m_command = SWAP32LE(head.m_command); - head.m_return_code = SWAP32LE(head.m_return_code); - head.m_flags = SWAP32LE(head.m_flags); - head.m_protocol_version = SWAP32LE(head.m_protocol_version); - std::string send_buff((const char*)&head, sizeof(head)); -#endif - send_buff += return_buff; - CRITICAL_REGION_BEGIN(m_send_lock); - if(!m_pservice_endpoint->do_send(send_buff.data(), send_buff.size())) + const uint32_t return_code = m_config.m_pcommands_handler->invoke( + m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + ); + + bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); + head.m_return_code = SWAP32LE(return_code); + return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head)); + + if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)})) return false; - CRITICAL_REGION_END(); - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << m_current_head.m_cb - << ", flags" << m_current_head.m_flags - << ", r?=" << m_current_head.m_have_to_return_data - <<", cmd = " << m_current_head.m_command - << ", ver=" << m_current_head.m_protocol_version); + + MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb + << ", flags" << head.m_flags + << ", r?=" << head.m_have_to_return_data + <<", cmd = " << head.m_command + << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); } + // reuse small buffer + if (!temp.empty() && temp.capacity() <= 64 * 1024) + { + temp.clear(); + m_fragment_buffer = std::move(temp); + } } - m_state = stream_state_head; break; case stream_state_head: { @@ -584,26 +626,10 @@ public: break; } - bucket_head2 head = {0}; - head.m_signature = SWAP64LE(LEVIN_SIGNATURE); - head.m_cb = SWAP64LE(in_buff.size()); - head.m_have_to_return_data = true; - - head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); - head.m_command = SWAP32LE(command); - head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); - CRITICAL_REGION_BEGIN(m_send_lock); - CRITICAL_REGION_LOCAL1(m_invoke_response_handlers_lock); - if(!m_pservice_endpoint->do_send(&head, sizeof(head))) - { - LOG_ERROR_CC(m_connection_context, "Failed to do_send"); - err_code = LEVIN_ERROR_CONNECTION; - break; - } + CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock); - if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) + if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -620,7 +646,7 @@ public: if (LEVIN_OK != err_code) { - epee::span<const uint8_t> stub_buff{(const uint8_t*)"", 0}; + epee::span<const uint8_t> stub_buff = nullptr; // Never call callback inside critical section, that can cause deadlock cb(err_code, stub_buff, m_connection_context); return false; @@ -642,35 +668,13 @@ public: if(m_deletion_initiated) return LEVIN_ERROR_CONNECTION_DESTROYED; - bucket_head2 head = {0}; - head.m_signature = SWAP64LE(LEVIN_SIGNATURE); - head.m_cb = SWAP64LE(in_buff.size()); - head.m_have_to_return_data = true; - - head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); - head.m_command = SWAP32LE(command); - head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); - CRITICAL_REGION_BEGIN(m_send_lock); - if(!m_pservice_endpoint->do_send(&head, sizeof(head))) - { - LOG_ERROR_CC(m_connection_context, "Failed to do_send"); - return LEVIN_ERROR_CONNECTION; - } - if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) + if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { - LOG_ERROR_CC(m_connection_context, "Failed to do_send"); + LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; } - CRITICAL_REGION_END(); - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", f=" << head.m_flags - << ", r?=" << head.m_have_to_return_data - << ", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); uint64_t ticks_start = misc_utils::get_tick_count(); size_t prev_size = 0; @@ -716,33 +720,38 @@ public: if(m_deletion_initiated) return LEVIN_ERROR_CONNECTION_DESTROYED; - bucket_head2 head = {0}; - head.m_signature = SWAP64LE(LEVIN_SIGNATURE); - head.m_have_to_return_data = false; - head.m_cb = SWAP64LE(in_buff.size()); - - head.m_command = SWAP32LE(command); - head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); - head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); - CRITICAL_REGION_BEGIN(m_send_lock); - if(!m_pservice_endpoint->do_send(&head, sizeof(head))) + if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) { - LOG_ERROR_CC(m_connection_context, "Failed to do_send()"); + LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); return -1; } - if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) + return 1; + } + + /*! Sends `message` without adding a levin header. The message must have + been created with `make_notify`, `make_noise_notify` or + `make_fragmented_notify`. See additional instructions for + `make_fragmented_notify`. + + \return 1 on success */ + int send(byte_slice message) + { + const misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler::finish_outer_call, this) + ); + + if(m_deletion_initiated) + return LEVIN_ERROR_CONNECTION_DESTROYED; + + const std::size_t length = message.size(); + if (!m_pservice_endpoint->do_send(std::move(message))) { - LOG_ERROR_CC(m_connection_context, "Failed to do_send()"); + LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - CRITICAL_REGION_END(); - LOG_DEBUG_CC(m_connection_context, "LEVIN_PACKET_SENT. [len=" << head.m_cb << - ", f=" << head.m_flags << - ", r?=" << head.m_have_to_return_data << - ", cmd = " << head.m_command << - ", ver=" << head.m_protocol_version); + MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -782,7 +791,7 @@ void async_protocol_handler_config<t_connection_context>::delete_connections(siz auto i = connections.end() - 1; async_protocol_handler<t_connection_context> *conn = m_connects.at(*i); del_connection(conn); - close(*i); + conn->close(); connections.erase(i); } catch (const std::out_of_range &e) @@ -923,6 +932,14 @@ int async_protocol_handler_config<t_connection_context>::notify(int command, con } //------------------------------------------------------------------------------------------ template<class t_connection_context> +int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) +{ + async_protocol_handler<t_connection_context>* aph; + int r = find_and_lock_connection(connection_id, aph); + return LEVIN_OK == r ? aph->send(std::move(message)) : 0; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> bool async_protocol_handler_config<t_connection_context>::close(boost::uuids::uuid connection_id) { CRITICAL_REGION_LOCAL(m_connects_lock); diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index ce74e1cd3..246cf6ad8 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -30,6 +30,11 @@ #include <string> #include <boost/algorithm/string/predicate.hpp> #include <boost/asio/ip/address_v6.hpp> +#include "int-util.h" + +// IP addresses are kept in network byte order +// Masks below are little endian +// -> convert from network byte order to host byte order before comparing namespace epee { @@ -62,6 +67,7 @@ namespace epee inline bool is_ip_local(uint32_t ip) { + ip = SWAP32LE(ip); /* local ip area 10.0.0.0 — 10.255.255.255 @@ -85,6 +91,7 @@ namespace epee inline bool is_ip_loopback(uint32_t ip) { + ip = SWAP32LE(ip); if( (ip | 0xffffff00) == 0xffffff7f) return true; //MAKE_IP diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index e315555fc..2b02eafa4 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -31,6 +31,7 @@ //#include <Winsock2.h> //#include <Ws2tcpip.h> +#include <atomic> #include <string> #include <boost/version.hpp> #include <boost/asio/io_service.hpp> @@ -154,7 +155,7 @@ namespace net_utils } inline - try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, epee::net_utils::ssl_support_t ssl_support) + try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) { m_deadline.expires_from_now(timeout); boost::unique_future<boost::asio::ip::tcp::socket> connection = m_connector(addr, port, m_deadline); @@ -174,11 +175,11 @@ namespace net_utils m_connected = true; m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); // SSL Options - if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr)) { - if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { boost::system::error_code ignored_ec; m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); @@ -217,7 +218,7 @@ namespace net_utils // Get a list of endpoints corresponding to the server name. - try_connect_result_t try_connect_result = try_connect(addr, port, timeout, m_ssl_options.support); + try_connect_result_t try_connect_result = try_connect(addr, port, timeout); if (try_connect_result == CONNECT_FAILURE) return false; if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) @@ -226,7 +227,7 @@ namespace net_utils { MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); m_ssl_options.support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; - if (try_connect(addr, port, timeout, m_ssl_options.support) != CONNECT_SUCCESS) + if (try_connect(addr, port, timeout) != CONNECT_SUCCESS) return false; } } @@ -562,7 +563,7 @@ namespace net_utils { m_deadline.cancel(); boost::system::error_code ec; - if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) + if(m_ssl_options) shutdown_ssl(); m_ssl_socket->next_layer().cancel(ec); if(ec) diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 5ae3e53b3..028e605d7 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -34,9 +34,11 @@ #include <boost/asio/ip/address_v6.hpp> #include <typeinfo> #include <type_traits> +#include "byte_slice.h" #include "enums.h" -#include "serialization/keyvalue_serialization.h" #include "misc_log_ex.h" +#include "serialization/keyvalue_serialization.h" +#include "int-util.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -90,7 +92,20 @@ namespace net_utils static constexpr bool is_blockable() noexcept { return true; } BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_ip) + if (is_store) + { + KV_SERIALIZE_VAL_POD_AS_BLOB_N(m_ip, "ip") + uint32_t ip = SWAP32LE(this_ref.m_ip); + epee::serialization::selector<is_store>::serialize(ip, stg, hparent_section, "m_ip"); + } + else + { + if (!epee::serialization::selector<is_store>::serialize_t_val_as_blob(this_ref.m_ip, stg, hparent_section, "ip")) + { + KV_SERIALIZE(m_ip) + const_cast<ipv4_network_address&>(this_ref).m_ip = SWAP32LE(this_ref.m_ip); + } + } KV_SERIALIZE(m_port) END_KV_SERIALIZE_MAP() }; @@ -424,7 +439,7 @@ namespace net_utils /************************************************************************/ struct i_service_endpoint { - virtual bool do_send(const void* ptr, size_t cb)=0; + virtual bool do_send(byte_slice message)=0; virtual bool close()=0; virtual bool send_done()=0; virtual bool call_run_once_service_io()=0; diff --git a/contrib/epee/include/readline_buffer.h b/contrib/epee/include/readline_buffer.h index 5968d243d..e8f75a9e1 100644 --- a/contrib/epee/include/readline_buffer.h +++ b/contrib/epee/include/readline_buffer.h @@ -40,5 +40,7 @@ namespace rdln readline_buffer* m_buffer; bool m_restart; }; + + void clear_screen(); } diff --git a/contrib/epee/include/storages/portable_storage_bin_utils.h b/contrib/epee/include/storages/portable_storage_bin_utils.h new file mode 100644 index 000000000..bcde64487 --- /dev/null +++ b/contrib/epee/include/storages/portable_storage_bin_utils.h @@ -0,0 +1,46 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "int-util.h" + +template<typename T> T convert_swapper(T t) { return t; } +template<> inline uint16_t convert_swapper(uint16_t t) { return SWAP16LE(t); } +template<> inline int16_t convert_swapper(int16_t t) { return SWAP16LE((uint16_t&)t); } +template<> inline uint32_t convert_swapper(uint32_t t) { return SWAP32LE(t); } +template<> inline int32_t convert_swapper(int32_t t) { return SWAP32LE((uint32_t&)t); } +template<> inline uint64_t convert_swapper(uint64_t t) { return SWAP64LE(t); } +template<> inline int64_t convert_swapper(int64_t t) { return SWAP64LE((uint64_t&)t); } +template<> inline double convert_swapper(double t) { union { uint64_t u; double d; } u; u.d = t; u.u = SWAP64LE(u.u); return u.d; } + +#if BYTE_ORDER == BIG_ENDIAN +#define CONVERT_POD(x) convert_swapper(x) +#else +#define CONVERT_POD(x) (x) +#endif diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index e0a32b3ca..c0b6cc7b1 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -30,6 +30,7 @@ #include "misc_language.h" #include "portable_storage_base.h" +#include "portable_storage_bin_utils.h" #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT @@ -117,6 +118,7 @@ namespace epee RECURSION_LIMITATION(); static_assert(std::is_pod<t_pod_type>::value, "POD type expected"); read(&pod_val, sizeof(pod_val)); + pod_val = CONVERT_POD(pod_val); } template<class t_type> @@ -140,7 +142,7 @@ namespace epee sa.reserve(size); //TODO: add some optimization here later while(size--) - sa.m_array.push_back(read<type_name>()); + sa.m_array.push_back(read<type_name>()); return storage_entry(array_entry(sa)); } diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 9501bbc2a..137497e19 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -31,6 +31,7 @@ #include "pragma_comp_defs.h" #include "misc_language.h" #include "portable_storage_base.h" +#include "portable_storage_bin_utils.h" namespace epee { @@ -40,8 +41,9 @@ namespace epee template<class pack_value, class t_stream> size_t pack_varint_t(t_stream& strm, uint8_t type_or, size_t& pv) { - pack_value v = (*((pack_value*)&pv)) << 2; + pack_value v = pv << 2; v |= type_or; + v = CONVERT_POD(v); strm.write((const char*)&v, sizeof(pack_value)); return sizeof(pack_value); } @@ -93,8 +95,11 @@ namespace epee uint8_t type = contained_type|SERIALIZE_FLAG_ARRAY; m_strm.write((const char*)&type, 1); pack_varint(m_strm, arr_pod.m_array.size()); - for(const t_pod_type& x: arr_pod.m_array) + for(t_pod_type x: arr_pod.m_array) + { + x = CONVERT_POD(x); m_strm.write((const char*)&x, sizeof(t_pod_type)); + } return true; } @@ -147,7 +152,8 @@ namespace epee bool pack_pod_type(uint8_t type, const pod_type& v) { m_strm.write((const char*)&type, 1); - m_strm.write((const char*)&v, sizeof(pod_type)); + pod_type v0 = CONVERT_POD(v); + m_strm.write((const char*)&v0, sizeof(pod_type)); return true; } //section, array_entry diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index 9f2404856..dba02f270 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -150,81 +150,6 @@ namespace epee }; -#if defined(WINDWOS_PLATFORM) - class shared_critical_section - { - public: - shared_critical_section() - { - ::InitializeSRWLock(&m_srw_lock); - } - ~shared_critical_section() - {} - - bool lock_shared() - { - AcquireSRWLockShared(&m_srw_lock); - return true; - } - bool unlock_shared() - { - ReleaseSRWLockShared(&m_srw_lock); - return true; - } - bool lock_exclusive() - { - ::AcquireSRWLockExclusive(&m_srw_lock); - return true; - } - bool unlock_exclusive() - { - ::ReleaseSRWLockExclusive(&m_srw_lock); - return true; - } - private: - SRWLOCK m_srw_lock; - }; - - - class shared_guard - { - public: - shared_guard(shared_critical_section& ref_sec):m_ref_sec(ref_sec) - { - m_ref_sec.lock_shared(); - } - - ~shared_guard() - { - m_ref_sec.unlock_shared(); - } - - private: - shared_critical_section& m_ref_sec; - }; - - - class exclusive_guard - { - public: - exclusive_guard(shared_critical_section& ref_sec):m_ref_sec(ref_sec) - { - m_ref_sec.lock_exclusive(); - } - - ~exclusive_guard() - { - m_ref_sec.unlock_exclusive(); - } - - private: - shared_critical_section& m_ref_sec; - }; -#endif - -#define SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard critical_region_var(x) -#define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard critical_region_var(x) - #define CRITICAL_REGION_LOCAL(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var(x) #define CRITICAL_REGION_BEGIN(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)> critical_region_var(x) #define CRITICAL_REGION_LOCAL1(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var1(x) @@ -232,22 +157,6 @@ namespace epee #define CRITICAL_REGION_END() } - -#if defined(WINDWOS_PLATFORM) - inline const char* get_wait_for_result_as_text(DWORD res) - { - switch(res) - { - case WAIT_ABANDONED: return "WAIT_ABANDONED"; - case WAIT_TIMEOUT: return "WAIT_TIMEOUT"; - case WAIT_OBJECT_0: return "WAIT_OBJECT_0"; - case WAIT_OBJECT_0+1: return "WAIT_OBJECT_1"; - case WAIT_OBJECT_0+2: return "WAIT_OBJECT_2"; - default: return "UNKNOWN CODE"; - } - } -#endif - } #endif diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 2465afebb..c512e3b86 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,8 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c - connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp) +add_library(epee STATIC byte_slice.cpp hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp + levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp) if (USE_READLINE AND GNU_READLINE_FOUND) add_library(epee_readline STATIC readline_buffer.cpp) diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp new file mode 100644 index 000000000..216049e5b --- /dev/null +++ b/contrib/epee/src/byte_slice.cpp @@ -0,0 +1,209 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <atomic> +#include <cstdlib> +#include <cstring> +#include <limits> +#include <stdexcept> +#include <utility> + +#include "byte_slice.h" + +namespace epee +{ + struct byte_slice_data + { + byte_slice_data() noexcept + : ref_count(1) + {} + + virtual ~byte_slice_data() noexcept + {} + + std::atomic<std::size_t> ref_count; + }; + + void release_byte_slice::operator()(byte_slice_data* ptr) const noexcept + { + if (ptr && --(ptr->ref_count) == 0) + { + ptr->~byte_slice_data(); + free(ptr); + } + } + + namespace + { + template<typename T> + struct adapted_byte_slice final : byte_slice_data + { + explicit adapted_byte_slice(T&& buffer) + : byte_slice_data(), buffer(std::move(buffer)) + {} + + virtual ~adapted_byte_slice() noexcept final override + {} + + const T buffer; + }; + + // bytes "follow" this structure in memory slab + struct raw_byte_slice final : byte_slice_data + { + raw_byte_slice() noexcept + : byte_slice_data() + {} + + virtual ~raw_byte_slice() noexcept final override + {} + }; + + /* This technique is not-standard, but allows for the reference count and + memory for the bytes (when given a list of spans) to be allocated in a + single call. In that situation, the dynamic sized bytes are after/behind + the raw_byte_slice class. The C runtime has to track the number of bytes + allocated regardless, so free'ing is relatively easy. */ + + template<typename T, typename... U> + std::unique_ptr<T, release_byte_slice> allocate_slice(std::size_t extra_bytes, U&&... args) + { + if (std::numeric_limits<std::size_t>::max() - sizeof(T) < extra_bytes) + throw std::bad_alloc{}; + + void* const ptr = malloc(sizeof(T) + extra_bytes); + if (ptr == nullptr) + throw std::bad_alloc{}; + + try + { + new (ptr) T{std::forward<U>(args)...}; + } + catch (...) + { + free(ptr); + throw; + } + return std::unique_ptr<T, release_byte_slice>{reinterpret_cast<T*>(ptr)}; + } + } // anonymous + + byte_slice::byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept + : storage_(storage), portion_(portion) + { + if (storage_) + ++(storage_->ref_count); + } + + template<typename T> + byte_slice::byte_slice(const adapt_buffer, T&& buffer) + : storage_(nullptr), portion_(to_byte_span(to_span(buffer))) + { + if (!buffer.empty()) + storage_ = allocate_slice<adapted_byte_slice<T>>(0, std::move(buffer)); + } + + byte_slice::byte_slice(std::initializer_list<span<const std::uint8_t>> sources) + : byte_slice() + { + std::size_t space_needed = 0; + for (const auto source : sources) + space_needed += source.size(); + + if (space_needed) + { + auto storage = allocate_slice<raw_byte_slice>(space_needed); + span<std::uint8_t> out{reinterpret_cast<std::uint8_t*>(storage.get() + 1), space_needed}; + portion_ = {out.data(), out.size()}; + + for (const auto source : sources) + { + std::memcpy(out.data(), source.data(), source.size()); + if (out.remove_prefix(source.size()) < source.size()) + throw std::bad_alloc{}; // size_t overflow on space_needed + } + storage_ = std::move(storage); + } + } + + byte_slice::byte_slice(std::string&& buffer) + : byte_slice(adapt_buffer{}, std::move(buffer)) + {} + + byte_slice::byte_slice(std::vector<std::uint8_t>&& buffer) + : byte_slice(adapt_buffer{}, std::move(buffer)) + {} + + byte_slice::byte_slice(byte_slice&& source) noexcept + : storage_(std::move(source.storage_)), portion_(source.portion_) + { + source.portion_ = epee::span<const std::uint8_t>{}; + } + + byte_slice& byte_slice::operator=(byte_slice&& source) noexcept + { + storage_ = std::move(source.storage_); + portion_ = source.portion_; + if (source.storage_ == nullptr) + source.portion_ = epee::span<const std::uint8_t>{}; + + return *this; + } + + std::size_t byte_slice::remove_prefix(std::size_t max_bytes) noexcept + { + max_bytes = portion_.remove_prefix(max_bytes); + if (portion_.empty()) + storage_ = nullptr; + return max_bytes; + } + + byte_slice byte_slice::take_slice(const std::size_t max_bytes) noexcept + { + byte_slice out{}; + std::uint8_t const* const ptr = data(); + out.portion_ = {ptr, portion_.remove_prefix(max_bytes)}; + + if (portion_.empty()) + out.storage_ = std::move(storage_); // no atomic inc/dec + else + out = {storage_.get(), out.portion_}; + + return out; + } + + byte_slice byte_slice::get_slice(const std::size_t begin, const std::size_t end) const + { + if (end < begin || portion_.size() < end) + throw std::out_of_range{"bad slice range"}; + + if (begin == end) + return {}; + return {storage_.get(), {portion_.begin() + begin, end - begin}}; + } +} // epee diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index 82d9e3b53..7526dde26 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -128,7 +128,7 @@ connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_thro int connection_basic_pimpl::m_default_tos; // methods: -connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) +connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) : m_state(std::move(state)), mI( new connection_basic_pimpl("peer") ), @@ -152,7 +152,7 @@ connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::s _note("Spawned connection #"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count); } -connection_basic::connection_basic(boost::asio::io_service &io_service, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) +connection_basic::connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) : m_state(std::move(state)), mI( new connection_basic_pimpl("peer") ), diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp new file mode 100644 index 000000000..ff845e2a7 --- /dev/null +++ b/contrib/epee/src/levin_base.cpp @@ -0,0 +1,128 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/levin_base.h" + +#include "int-util.h" + +namespace epee +{ +namespace levin +{ + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept + { + bucket_head2 head = {0}; + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_have_to_return_data = expect_response; + head.m_cb = SWAP64LE(msg_size); + + head.m_command = SWAP32LE(command); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); + head.m_flags = SWAP32LE(flags); + return head; + } + + byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) + { + const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); + return byte_slice{epee::as_byte_span(head), payload}; + } + + byte_slice make_noise_notify(const std::size_t noise_bytes) + { + static constexpr const std::uint32_t flags = + LEVIN_PACKET_BEGIN | LEVIN_PACKET_END; + + if (noise_bytes < sizeof(bucket_head2)) + return nullptr; + + std::string buffer(noise_bytes, char(0)); + const bucket_head2 head = make_header(0, noise_bytes - sizeof(bucket_head2), flags, false); + std::memcpy(std::addressof(buffer[0]), std::addressof(head), sizeof(head)); + + return byte_slice{std::move(buffer)}; + } + + byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + { + const size_t noise_size = noise_message.size(); + if (noise_size < sizeof(bucket_head2) * 2) + return nullptr; + + if (payload.size() <= noise_size - sizeof(bucket_head2)) + { + /* The entire message can be sent at once, and the levin binary parser + will ignore extra bytes. So just pad with zeroes and otherwise send + a "normal", not fragmented message. */ + const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); + const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; + + const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); + return byte_slice{as_byte_span(head), payload, padding_bytes}; + } + + // fragment message + const size_t payload_space = noise_size - sizeof(bucket_head2); + const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; + + std::string buffer{}; + buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + + bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); + buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + + head.m_command = command; + head.m_flags = LEVIN_PACKET_REQUEST; + head.m_cb = payload.size(); + buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + + size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); + buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + + head.m_command = 0; + head.m_flags = 0; + head.m_cb = noise_size - sizeof(bucket_head2); + + while (!payload.empty()) + { + copy_size = payload.remove_prefix(payload_space); + + if (payload.empty()) + head.m_flags = LEVIN_PACKET_END; + + buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + } + + const size_t padding = noise_size - copy_size - sizeof(bucket_head2); + buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + + return byte_slice{std::move(buffer)}; + } +} // levin +} // epee diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index 39369c43f..05322b693 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -71,6 +71,11 @@ rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const { boost::lock_guard<boost::mutex> lock(sync_mutex); line_stat = rdln::partial; + if (!m_cout_buf) + { + line = ""; + return rdln::full; + } rl_callback_read_char(); if (line_stat == rdln::full) { @@ -224,3 +229,8 @@ static void remove_line_handler() rl_callback_handler_remove(); } +void rdln::clear_screen() +{ + rl_clear_screen(0, 0); +} + diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index b654b15c7..5913fda3a 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -54,7 +54,7 @@ def build(): print('\nCompiling ' + args.version + ' Linux') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) - subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero-*.tar.bz2 ../monero-binaries/'+args.version, shell=True) if args.windows: print('\nCompiling ' + args.version + ' Windows') @@ -64,9 +64,9 @@ def build(): if args.macos: print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero*.tar.bz2 ../monero-binaries/'+args.version, shell=True) os.chdir(workdir) @@ -141,8 +141,8 @@ def main(): if not 'LXC_GUEST_IP' in os.environ.keys(): os.environ['LXC_GUEST_IP'] = '10.0.3.5' - # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + # Disable MacOS build if no SDK found + if args.build and args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index fd94d43bf..8cbde2acb 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -164,7 +164,7 @@ script: | make ${MAKEOPTS} DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 cd .. rm -rf build done diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 77ea30072..d3141e2c7 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -111,7 +111,7 @@ script: | make ${MAKEOPTS} DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 cd .. rm -rf build done diff --git a/external/boost/archive/portable_binary_iarchive.hpp b/external/boost/archive/portable_binary_iarchive.hpp index bd19599f3..d17e95c05 100644 --- a/external/boost/archive/portable_binary_iarchive.hpp +++ b/external/boost/archive/portable_binary_iarchive.hpp @@ -258,7 +258,7 @@ portable_binary_iarchive::load_impl(boost::intmax_t & l, char maxsize){ this->primitive_base_t::load_binary(cptr, size);
#if BOOST_ENDIAN_BIG_BYTE
- if(m_flags & endian_little)
+ if((m_flags & endian_little) || (!(m_flags & endian_big)))
#else
if(m_flags & endian_big)
#endif
@@ -343,6 +343,8 @@ portable_binary_iarchive::init(unsigned int flags){ );
#endif
}
+ if (!(m_flags & (endian_little | endian_big)))
+ m_flags |= endian_little;
unsigned char x;
load(x);
m_flags = x << CHAR_BIT;
diff --git a/external/boost/archive/portable_binary_oarchive.hpp b/external/boost/archive/portable_binary_oarchive.hpp index 783c7f7c9..a0ac0a9b5 100644 --- a/external/boost/archive/portable_binary_oarchive.hpp +++ b/external/boost/archive/portable_binary_oarchive.hpp @@ -171,7 +171,7 @@ protected: void init(unsigned int flags);
public:
- portable_binary_oarchive(std::ostream & os, unsigned flags = 0) :
+ portable_binary_oarchive(std::ostream & os, unsigned flags = endian_little) :
primitive_base_t(
* os.rdbuf(),
0 != (flags & boost::archive::no_codecvt)
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 1ede7af62..760e380a9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -595,7 +595,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const MDEBUG("Space remaining: " << mei.me_mapsize - size_used); MDEBUG("Size threshold: " << threshold_size); float resize_percent = RESIZE_PERCENT; - MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % (100.*size_used/mei.me_mapsize) % (100.*resize_percent)); if (threshold_size > 0) { diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index f824d93a6..857e97afd 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -415,6 +415,115 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id return fret; } +static bool for_all_transactions(const std::string &filename, const uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(bool, uint64_t, const cryptonote::transaction_prefix&)> &f) +{ + MDB_env *env; + MDB_dbi dbi_blocks, dbi_txs; + MDB_txn *txn; + MDB_cursor *cur_blocks, *cur_txs; + int dbr; + bool tx_active = false; + MDB_val k; + MDB_val v; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 3); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "blocks", MDB_INTEGERKEY, &dbi_blocks); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_dbi_open(txn, "txs_pruned", MDB_INTEGERKEY, &dbi_txs); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_cursor_open(txn, dbi_blocks, &cur_blocks); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_txs, &cur_txs); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_stat stat; + dbr = mdb_stat(txn, dbi_blocks, &stat); + if (dbr) throw std::runtime_error("Failed to query txs stat: " + std::string(mdb_strerror(dbr))); + uint64_t n_blocks = stat.ms_entries; + dbr = mdb_stat(txn, dbi_txs, &stat); + if (dbr) throw std::runtime_error("Failed to query txs stat: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; + + bool fret = true; + + MDB_cursor_op op_blocks = MDB_FIRST; + MDB_cursor_op op_txs = MDB_FIRST; + uint64_t tx_idx = 0; + while (1) + { + int ret = mdb_cursor_get(cur_blocks, &k, &v, op_blocks); + op_blocks = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate blocks: " + std::string(mdb_strerror(ret))); + + if (k.mv_size != sizeof(uint64_t)) + throw std::runtime_error("Bad key size"); + uint64_t height = *(const uint64_t*)k.mv_data; + blobdata bd; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw std::runtime_error("Failed to parse block from blob retrieved from the db"); + + ret = mdb_cursor_get(cur_txs, &k, &v, op_txs); + if (ret) + throw std::runtime_error("Failed to fetch transaction " + string_tools::pod_to_hex(get_transaction_hash(b.miner_tx)) + ": " + std::string(mdb_strerror(ret))); + op_txs = MDB_NEXT; + + bool last_block = height == n_blocks - 1; + if (start_idx <= tx_idx++ && !f(last_block && b.tx_hashes.empty(), height, b.miner_tx)) + { + fret = false; + break; + } + for (size_t i = 0; i < b.tx_hashes.size(); ++i) + { + const crypto::hash& txid = b.tx_hashes[i]; + ret = mdb_cursor_get(cur_txs, &k, &v, op_txs); + if (ret) + throw std::runtime_error("Failed to fetch transaction " + string_tools::pod_to_hex(txid) + ": " + std::string(mdb_strerror(ret))); + if (start_idx <= tx_idx++) + { + cryptonote::transaction_prefix tx; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + CHECK_AND_ASSERT_MES(parse_and_validate_tx_prefix_from_blob(bd, tx), false, "Failed to parse transaction from blob"); + if (!f(last_block && i == b.tx_hashes.size() - 1, height, tx)) + { + fret = false; + break; + } + } + } + if (!fret) + break; + } + + mdb_cursor_close(cur_blocks); + mdb_cursor_close(cur_txs); + mdb_txn_commit(txn); + tx_active = false; + mdb_dbi_close(env, dbi_blocks); + mdb_dbi_close(env, dbi_txs); + mdb_env_close(env); + return fret; +} + static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) { MDB_env *env[2]; @@ -1094,6 +1203,7 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"}; const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; + const command_line::arg_descriptor<bool> arg_historical_stat = {"historical-stat", "Report historical stat of spent outputs for every 10000 blocks window"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); command_line::add_arg(desc_cmd_sett, arg_log_level); @@ -1105,6 +1215,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); command_line::add_arg(desc_cmd_sett, arg_export); command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); + command_line::add_arg(desc_cmd_sett, arg_historical_stat); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -1145,6 +1256,7 @@ int main(int argc, char* argv[]) bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); bool opt_verbose = command_line::get_arg(vm, arg_verbose); bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + bool opt_historical_stat = command_line::get_arg(vm, arg_historical_stat); std::string opt_export = command_line::get_arg(vm, arg_export); std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list); @@ -1196,6 +1308,69 @@ int main(int argc, char* argv[]) MDB_cursor *cur0; open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); + std::vector<output_data> work_spent; + + if (opt_historical_stat) + { + if (!start_blackballed_outputs) + { + MINFO("Spent outputs database is empty. Either you haven't run the analysis mode yet, or there is really no output marked as spent."); + goto skip_secondary_passes; + } + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + const uint64_t STAT_WINDOW = 10000; + uint64_t outs_total = 0; + uint64_t outs_spent = 0; + std::unordered_map<uint64_t, uint64_t> outs_per_amount; + uint64_t start_idx = 0, n_txes; + uint64_t prev_height = 0; + for_all_transactions(inputs[0], start_idx, n_txes, [&](bool last_tx, uint64_t height, const cryptonote::transaction_prefix &tx)->bool + { + if (height != prev_height) + { + if (height % 100 == 0) std::cout << "\r" << height << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " ) \r" << std::flush; + if (height % STAT_WINDOW == 0) + { + uint64_t window_front = (height / STAT_WINDOW - 1) * STAT_WINDOW; + uint64_t window_back = window_front + STAT_WINDOW - 1; + LOG_PRINT_L0(window_front << "-" << window_back << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " )"); + outs_total = outs_spent = 0; + } + } + prev_height = height; + for (const auto &out: tx.vout) + { + ++outs_total; + CHECK_AND_ASSERT_THROW_MES(out.target.type() == typeid(txout_to_key), "Out target type is not txout_to_key: height=" + std::to_string(height)); + uint64_t out_global_index = outs_per_amount[out.amount]++; + if (is_output_spent(cur, output_data(out.amount, out_global_index))) + ++outs_spent; + } + if (last_tx) + { + uint64_t window_front = (height / STAT_WINDOW) * STAT_WINDOW; + uint64_t window_back = height; + LOG_PRINT_L0(window_front << "-" << window_back << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " )"); + } + if (stop_requested) + { + MINFO("Stopping scan..."); + return false; + } + return true; + }); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + goto skip_secondary_passes; + } + if (!extra_spent_outputs.empty()) { MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); @@ -1432,8 +1607,6 @@ int main(int argc, char* argv[]) break; } - std::vector<output_data> work_spent; - if (stop_requested) goto skip_secondary_passes; diff --git a/src/common/util.cpp b/src/common/util.cpp index 0fa9e8dc1..57e747837 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -30,6 +30,7 @@ #include <unistd.h> #include <cstdio> +#include <wchar.h> #ifdef __GLIBC__ #include <gnu/libc-version.h> @@ -67,6 +68,7 @@ using namespace epee; #include "memwipe.h" #include "cryptonote_config.h" #include "net/http_client.h" // epee::net_utils::... +#include "readline_buffer.h" #ifdef WIN32 #ifndef STRSAFE_NO_DEPRECATE @@ -1119,4 +1121,162 @@ std::string get_nix_version_display_string() return (boost::format(size->format) % (double(bytes) / divisor)).str(); } + void clear_screen() + { + std::cout << "\033[2K" << std::flush; // clear whole line + std::cout << "\033c" << std::flush; // clear current screen and scrollback + std::cout << "\033[2J" << std::flush; // clear current screen only, scrollback is still around + std::cout << "\033[3J" << std::flush; // does nothing, should clear current screen and scrollback + std::cout << "\033[1;1H" << std::flush; // move cursor top/left + std::cout << "\r \r" << std::flush; // erase odd chars if the ANSI codes were printed raw +#ifdef _WIN32 + COORD coord{0, 0}; + CONSOLE_SCREEN_BUFFER_INFO csbi; + HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE); + if (GetConsoleScreenBufferInfo(h, &csbi)) + { + DWORD cbConSize = csbi.dwSize.X * csbi.dwSize.Y, w; + FillConsoleOutputCharacter(h, (TCHAR)' ', cbConSize, coord, &w); + if (GetConsoleScreenBufferInfo(h, &csbi)) + FillConsoleOutputAttribute(h, csbi.wAttributes, cbConSize, coord, &w); + SetConsoleCursorPosition(h, coord); + } +#endif + } + + std::pair<std::string, size_t> get_string_prefix_by_width(const std::string &s, size_t columns) + { + std::string sc = ""; + size_t avail = s.size(); + const char *ptr = s.data(); + wint_t cp = 0; + int bytes = 1; + size_t sw = 0; + char wbuf[8], *wptr; + while (avail--) + { + if ((*ptr & 0x80) == 0) + { + cp = *ptr++; + bytes = 1; + } + else if ((*ptr & 0xe0) == 0xc0) + { + if (avail < 1) + { + MERROR("Invalid UTF-8"); + return std::make_pair(s, s.size()); + } + cp = (*ptr++ & 0x1f) << 6; + cp |= *ptr++ & 0x3f; + --avail; + bytes = 2; + } + else if ((*ptr & 0xf0) == 0xe0) + { + if (avail < 2) + { + MERROR("Invalid UTF-8"); + return std::make_pair(s, s.size()); + } + cp = (*ptr++ & 0xf) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 2; + bytes = 3; + } + else if ((*ptr & 0xf8) == 0xf0) + { + if (avail < 3) + { + MERROR("Invalid UTF-8"); + return std::make_pair(s, s.size()); + } + cp = (*ptr++ & 0x7) << 18; + cp |= (*ptr++ & 0x3f) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 3; + bytes = 4; + } + else + { + MERROR("Invalid UTF-8"); + return std::make_pair(s, s.size()); + } + + wptr = wbuf; + switch (bytes) + { + case 1: *wptr++ = cp; break; + case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break; + case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + default: MERROR("Invalid UTF-8"); return std::make_pair(s, s.size()); + } + *wptr = 0; + sc += std::string(wbuf, bytes); +#ifdef _WIN32 + int cpw = 1; // Guess who does not implement wcwidth +#else + int cpw = wcwidth(cp); +#endif + if (cpw > 0) + { + if (cpw > (int)columns) + break; + columns -= cpw; + sw += cpw; + } + cp = 0; + bytes = 1; + } + return std::make_pair(sc, sw); + } + + size_t get_string_width(const std::string &s) + { + return get_string_prefix_by_width(s, 999999999).second; + }; + + std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns) + { + std::vector<std::string> words; + std::vector<std::pair<std::string, size_t>> lines; + boost::split(words, s, boost::is_any_of(" "), boost::token_compress_on); + // split large "words" + for (size_t i = 0; i < words.size(); ++i) + { + for (;;) + { + std::string prefix = get_string_prefix_by_width(words[i], columns).first; + if (prefix == words[i]) + break; + words[i] = words[i].substr(prefix.size()); + words.insert(words.begin() + i, prefix); + } + } + + lines.push_back(std::make_pair("", 0)); + while (!words.empty()) + { + const size_t word_len = get_string_width(words.front()); + size_t line_len = get_string_width(lines.back().first); + if (line_len > 0 && line_len + 1 + word_len > columns) + { + lines.push_back(std::make_pair("", 0)); + line_len = 0; + } + if (line_len > 0) + { + lines.back().first += " "; + lines.back().second++; + } + lines.back().first += words.front(); + lines.back().second += word_len; + words.erase(words.begin()); + } + return lines; + } + } diff --git a/src/common/util.h b/src/common/util.h index b0f734eff..b794d7908 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -248,4 +248,8 @@ namespace tools std::string get_human_readable_timespan(uint64_t seconds); std::string get_human_readable_bytes(uint64_t bytes); + + void clear_screen(); + + std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns); } diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index d4b2a22bc..0ec992de9 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -88,13 +88,24 @@ namespace crypto { return &reinterpret_cast<const unsigned char &>(scalar); } - void generate_random_bytes_thread_safe(size_t N, uint8_t *bytes) + boost::mutex &get_random_lock() { static boost::mutex random_lock; - boost::lock_guard<boost::mutex> lock(random_lock); + return random_lock; + } + + void generate_random_bytes_thread_safe(size_t N, uint8_t *bytes) + { + boost::lock_guard<boost::mutex> lock(get_random_lock()); generate_random_bytes_not_thread_safe(N, bytes); } + void add_extra_entropy_thread_safe(const void *ptr, size_t bytes) + { + boost::lock_guard<boost::mutex> lock(get_random_lock()); + add_extra_entropy_not_thread_safe(ptr, bytes); + } + static inline bool less32(const unsigned char *k0, const unsigned char *k1) { for (int n = 31; n >= 0; --n) diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index bac456f60..8ce321f71 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -147,6 +147,7 @@ namespace crypto { }; void generate_random_bytes_thread_safe(size_t N, uint8_t *bytes); + void add_extra_entropy_thread_safe(const void *ptr, size_t bytes); /* Generate N random bytes */ diff --git a/src/crypto/random.c b/src/crypto/random.c index 74b202661..766b5f558 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -146,3 +146,18 @@ void generate_random_bytes_not_thread_safe(size_t n, void *result) { } } } + +void add_extra_entropy_not_thread_safe(const void *ptr, size_t bytes) +{ + size_t i; + + while (bytes > 0) + { + hash_permutation(&state); + const size_t round_bytes = bytes > HASH_DATA_AREA ? HASH_DATA_AREA : bytes; + for (i = 0; i < round_bytes; ++i) + state.b[i] ^= ((const uint8_t*)ptr)[i]; + bytes -= round_bytes; + ptr = cpadd(ptr, round_bytes); + } +} diff --git a/src/crypto/random.h b/src/crypto/random.h index ccb9f4853..21a66d776 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -33,3 +33,4 @@ #include <stddef.h> void generate_random_bytes_not_thread_safe(size_t n, void *result); +void add_extra_entropy_not_thread_safe(const void *ptr, size_t bytes); diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 1fa819b57..647471513 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -136,8 +136,8 @@ static inline int use_v4_jit(void) { \ U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ - division_result = state.hs.w[12]; \ - sqrt_result = state.hs.w[13]; \ + division_result = SWAP64LE(state.hs.w[12]); \ + sqrt_result = SWAP64LE(state.hs.w[13]); \ } while (0) #define VARIANT2_PORTABLE_INIT() \ @@ -210,7 +210,7 @@ static inline int use_v4_jit(void) uint64_t b0[2]; \ memcpy_swap64le(b0, b, 2); \ chunk2[0] = SWAP64LE(chunk1_old[0] + b0[0]); \ - chunk2[1] = SWAP64LE(SWAP64LE(chunk1_old[1]) + b0[1]); \ + chunk2[1] = SWAP64LE(chunk1_old[1] + b0[1]); \ if (variant >= 4) \ { \ uint64_t out_copy[2]; \ diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 96398a90b..51076e8c0 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -31,8 +31,10 @@ #pragma once #include <unordered_set> #include <atomic> +#include <boost/date_time/posix_time/posix_time.hpp> #include "net/net_utils_base.h" #include "copyable_atomic.h" +#include "crypto/hash.h" namespace cryptonote { diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index 02ed89e5a..771deb04c 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -34,7 +34,6 @@ #include <vector> #include <string> #include <boost/multiprecision/cpp_int.hpp> - #include "crypto/hash.h" namespace cryptonote diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index b68bb41e1..173b454f6 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -100,6 +100,16 @@ #define CRYPTONOTE_MEMPOOL_TX_LIVETIME (86400*3) //seconds, three days #define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME 604800 //seconds, one week +// see src/cryptonote_protocol/levin_notify.cpp +#define CRYPTONOTE_NOISE_MIN_EPOCH 5 // minutes +#define CRYPTONOTE_NOISE_EPOCH_RANGE 30 // seconds +#define CRYPTONOTE_NOISE_MIN_DELAY 10 // seconds +#define CRYPTONOTE_NOISE_DELAY_RANGE 5 // seconds +#define CRYPTONOTE_NOISE_BYTES 3*1024 // 3 KiB +#define CRYPTONOTE_NOISE_CHANNELS 2 // Max outgoing connections per zone used for noise/covert sending + +#define CRYPTONOTE_MAX_FRAGMENTS 20 // ~20 * NOISE_BYTES max payload size for covert/noise send + #define COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT 1000 #define P2P_LOCAL_WHITE_PEERLIST_LIMIT 1000 diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index dab1a0a96..8ed0e526a 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1738,7 +1738,8 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id const uint64_t prev_height = alt_chain.size() ? prev_data.height : m_db->get_block_height(b.prev_id); bei.height = prev_height + 1; uint64_t block_reward = get_outs_money_amount(b.miner_tx); - bei.already_generated_coins = block_reward + (alt_chain.size() ? prev_data.already_generated_coins : m_db->get_block_already_generated_coins(prev_height)); + const uint64_t prev_generated_coins = alt_chain.size() ? prev_data.already_generated_coins : m_db->get_block_already_generated_coins(prev_height); + bei.already_generated_coins = (block_reward < (MONEY_SUPPLY - prev_generated_coins)) ? prev_generated_coins + block_reward : MONEY_SUPPLY; // verify that the block's timestamp is within the acceptable range // (not earlier than the median of the last X blocks) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2ff7b5938..65115ee72 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -438,7 +438,7 @@ namespace cryptonote MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, "Received NOTIFY_NEW_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)"); if(context.m_state != cryptonote_connection_context::state_normal) return 1; - if(!is_synchronized()) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks + if(!is_synchronized() || m_no_sync) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks { LOG_DEBUG_CC(context, "Received new block while syncing, ignored"); return 1; @@ -508,7 +508,7 @@ namespace cryptonote MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, "Received NOTIFY_NEW_FLUFFY_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)"); if(context.m_state != cryptonote_connection_context::state_normal) return 1; - if(!is_synchronized()) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks + if(!is_synchronized() || m_no_sync) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks { LOG_DEBUG_CC(context, "Received new block while syncing, ignored"); return 1; @@ -899,7 +899,7 @@ namespace cryptonote // while syncing, core will lock for a long time, so we ignore // those txes as they aren't really needed anyway, and avoid a // long block before replying - if(!is_synchronized()) + if(!is_synchronized() || m_no_sync) { LOG_DEBUG_CC(context, "Received new tx while syncing, ignored"); return 1; @@ -2227,69 +2227,11 @@ skip: template<class t_core> bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) { - const bool hide_tx_broadcast = - 1 < m_p2p->get_zone_count() && exclude_context.m_remote_address.get_zone() == epee::net_utils::zone::invalid; - - if (hide_tx_broadcast) - MDEBUG("Attempting to conceal origin of tx via anonymity network connection(s)"); + for(auto& tx_blob : arg.txs) + m_core.on_transaction_relayed(tx_blob); // no check for success, so tell core they're relayed unconditionally - const bool pad_transactions = m_core.pad_transactions() || hide_tx_broadcast; - size_t bytes = pad_transactions ? 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(arg.txs.size()).size() : 0; - for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end(); ++tx_blob_it) - { - m_core.on_transaction_relayed(*tx_blob_it); - if (pad_transactions) - bytes += tools::get_varint_data(tx_blob_it->size()).size() + tx_blob_it->size(); - } - - if (pad_transactions) - { - // stuff some dummy bytes in to stay safe from traffic volume analysis - static constexpr size_t granularity = 1024; - size_t padding = granularity - bytes % granularity; - const size_t overhead = 2 /* 1 + '_' */ + tools::get_varint_data(padding).size(); - if (overhead > padding) - padding = 0; - else - padding -= overhead; - arg._ = std::string(padding, ' '); - - std::string arg_buff; - epee::serialization::store_t_to_binary(arg, arg_buff); - - // we probably lowballed the payload size a bit, so added a but too much. Fix this now. - size_t remove = arg_buff.size() % granularity; - if (remove > arg._.size()) - arg._.clear(); - else - arg._.resize(arg._.size() - remove); - // if the size of _ moved enough, we might lose byte in size encoding, we don't care - } - - std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections; - m_p2p->for_each_connection([hide_tx_broadcast, &exclude_context, &connections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) - { - const epee::net_utils::zone current_zone = context.m_remote_address.get_zone(); - const bool broadcast_to_peer = - peer_id && - (hide_tx_broadcast != bool(current_zone == epee::net_utils::zone::public_)) && - exclude_context.m_connection_id != context.m_connection_id; - - if (broadcast_to_peer) - connections.push_back({current_zone, context.m_connection_id}); - - return true; - }); - - if (connections.empty()) - MERROR("Transaction not relayed - no" << (hide_tx_broadcast ? " privacy": "") << " peers available"); - else - { - std::string fullBlob; - epee::serialization::store_t_to_binary(arg, fullBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<uint8_t>(fullBlob), std::move(connections)); - } + m_p2p->send_txs(std::move(arg.txs), exclude_context.m_remote_address.get_zone(), exclude_context.m_connection_id, m_core.pad_transactions()); return true; } //------------------------------------------------------------------------------------------------------------------------ diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp new file mode 100644 index 000000000..26cd93b5a --- /dev/null +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -0,0 +1,574 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "levin_notify.h" + +#include <boost/asio/steady_timer.hpp> +#include <boost/system/system_error.hpp> +#include <chrono> +#include <deque> +#include <stdexcept> + +#include "common/expect.h" +#include "common/varint.h" +#include "cryptonote_config.h" +#include "crypto/random.h" +#include "cryptonote_basic/connection_context.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "net/dandelionpp.h" +#include "p2p/net_node.h" + +namespace cryptonote +{ +namespace levin +{ + namespace + { + constexpr std::size_t connection_id_reserve_size = 100; + + constexpr const std::chrono::minutes noise_min_epoch{CRYPTONOTE_NOISE_MIN_EPOCH}; + constexpr const std::chrono::seconds noise_epoch_range{CRYPTONOTE_NOISE_EPOCH_RANGE}; + + constexpr const std::chrono::seconds noise_min_delay{CRYPTONOTE_NOISE_MIN_DELAY}; + constexpr const std::chrono::seconds noise_delay_range{CRYPTONOTE_NOISE_DELAY_RANGE}; + + /*! Select a randomized duration from 0 to `range`. The precision will be to + the systems `steady_clock`. As an example, supplying 3 seconds to this + function will select a duration from [0, 3] seconds, and the increments + for the selection will be determined by the `steady_clock` precision + (typically nanoseconds). + + \return A randomized duration from 0 to `range`. */ + std::chrono::steady_clock::duration random_duration(std::chrono::steady_clock::duration range) + { + using rep = std::chrono::steady_clock::rep; + return std::chrono::steady_clock::duration{crypto::rand_range(rep(0), range.count())}; + } + + //! \return All outgoing connections supporting fragments in `connections`. + std::vector<boost::uuids::uuid> get_out_connections(connections& p2p) + { + std::vector<boost::uuids::uuid> outs; + outs.reserve(connection_id_reserve_size); + + /* The foreach call is serialized with a lock, but should be quick due to + the reserve call so a strand is not used. Investigate if there is lots + of waiting in here. */ + + p2p.foreach_connection([&outs] (detail::p2p_context& context) { + if (!context.m_is_income) + outs.emplace_back(context.m_connection_id); + return true; + }); + + return outs; + } + + std::string make_tx_payload(std::vector<blobdata>&& txs, const bool pad) + { + NOTIFY_NEW_TRANSACTIONS::request request{}; + request.txs = std::move(txs); + + if (pad) + { + size_t bytes = 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(request.txs.size()).size(); + for(auto tx_blob_it = request.txs.begin(); tx_blob_it!=request.txs.end(); ++tx_blob_it) + bytes += tools::get_varint_data(tx_blob_it->size()).size() + tx_blob_it->size(); + + // stuff some dummy bytes in to stay safe from traffic volume analysis + static constexpr const size_t granularity = 1024; + size_t padding = granularity - bytes % granularity; + const size_t overhead = 2 /* 1 + '_' */ + tools::get_varint_data(padding).size(); + if (overhead > padding) + padding = 0; + else + padding -= overhead; + request._ = std::string(padding, ' '); + + std::string arg_buff; + epee::serialization::store_t_to_binary(request, arg_buff); + + // we probably lowballed the payload size a bit, so added a but too much. Fix this now. + size_t remove = arg_buff.size() % granularity; + if (remove > request._.size()) + request._.clear(); + else + request._.resize(request._.size() - remove); + // if the size of _ moved enough, we might lose byte in size encoding, we don't care + } + + std::string fullBlob; + if (!epee::serialization::store_t_to_binary(request, fullBlob)) + throw std::runtime_error{"Failed to serialize to epee binary format"}; + + return fullBlob; + } + + /* The current design uses `asio::strand`s. The documentation isn't as clear + as it should be - a `strand` has an internal `mutex` and `bool`. The + `mutex` synchronizes thread access and the `bool` is set when a thread is + executing something "in the strand". Therefore, if a callback has lots of + work to do in a `strand`, asio can switch to some other task instead of + blocking 1+ threads to wait for the original thread to complete the task + (as is the case when client code has a `mutex` inside the callback). The + downside is that asio _always_ allocates for the callback, even if it can + be immediately executed. So if all work in a strand is minimal, a lock + may be better. + + This code uses a strand per "zone" and a strand per "channel in a zone". + `dispatch` is used heavily, which means "execute immediately in _this_ + thread if the strand is not in use, otherwise queue the callback to be + executed immediately after the strand completes its current task". + `post` is used where deferred execution to an `asio::io_service::run` + thread is preferred. + + The strand per "zone" is useful because the levin + `foreach_connection` is blocked with a mutex anyway. So this primarily + helps with reducing blocking of a thread attempting a "flood" + notification. Updating/merging the outgoing connections in the + Dandelion++ map is also somewhat expensive. + + The strand per "channel" may need a re-visit. The most "expensive" code + is figuring out the noise/notification to send. If levin code is + optimized further, it might be better to just use standard locks per + channel. */ + + //! A queue of levin messages for a noise i2p/tor link + struct noise_channel + { + explicit noise_channel(boost::asio::io_service& io_service) + : active(nullptr), + queue(), + strand(io_service), + next_noise(io_service), + connection(boost::uuids::nil_uuid()) + {} + + // `asio::io_service::strand` cannot be copied or moved + noise_channel(const noise_channel&) = delete; + noise_channel& operator=(const noise_channel&) = delete; + + // Only read/write these values "inside the strand" + + epee::byte_slice active; + std::deque<epee::byte_slice> queue; + boost::asio::io_service::strand strand; + boost::asio::steady_timer next_noise; + boost::uuids::uuid connection; + }; + } // anonymous + + namespace detail + { + struct zone + { + explicit zone(boost::asio::io_service& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in) + : p2p(std::move(p2p)), + noise(std::move(noise_in)), + next_epoch(io_service), + strand(io_service), + map(), + channels(), + connection_count(0) + { + for (std::size_t count = 0; !noise.empty() && count < CRYPTONOTE_NOISE_CHANNELS; ++count) + channels.emplace_back(io_service); + } + + const std::shared_ptr<connections> p2p; + const epee::byte_slice noise; //!< `!empty()` means zone is using noise channels + boost::asio::steady_timer next_epoch; + boost::asio::io_service::strand strand; + net::dandelionpp::connection_map map;//!< Tracks outgoing uuid's for noise channels or Dandelion++ stems + std::deque<noise_channel> channels; //!< Never touch after init; only update elements on `noise_channel.strand` + std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time + }; + } // detail + + namespace + { + //! Adds a message to the sending queue of the channel. + class queue_covert_notify + { + std::shared_ptr<detail::zone> zone_; + epee::byte_slice message_; // Requires manual copy constructor + const std::size_t destination_; + + public: + queue_covert_notify(std::shared_ptr<detail::zone> zone, epee::byte_slice message, std::size_t destination) + : zone_(std::move(zone)), message_(std::move(message)), destination_(destination) + {} + + queue_covert_notify(queue_covert_notify&&) = default; + queue_covert_notify(const queue_covert_notify& source) + : zone_(source.zone_), message_(source.message_.clone()), destination_(source.destination_) + {} + + //! \pre Called within `zone_->channels[destionation_].strand`. + void operator()() + { + if (!zone_) + return; + + noise_channel& channel = zone_->channels.at(destination_); + assert(channel.strand.running_in_this_thread()); + + if (!channel.connection.is_nil()) + channel.queue.push_back(std::move(message_)); + } + }; + + //! Sends a message to every active connection + class flood_notify + { + std::shared_ptr<detail::zone> zone_; + epee::byte_slice message_; // Requires manual copy + boost::uuids::uuid source_; + + public: + explicit flood_notify(std::shared_ptr<detail::zone> zone, epee::byte_slice message, const boost::uuids::uuid& source) + : zone_(std::move(zone)), message_(message.clone()), source_(source) + {} + + flood_notify(flood_notify&&) = default; + flood_notify(const flood_notify& source) + : zone_(source.zone_), message_(source.message_.clone()), source_(source.source_) + {} + + void operator()() const + { + if (!zone_ || !zone_->p2p) + return; + + assert(zone_->strand.running_in_this_thread()); + + /* The foreach should be quick, but then it iterates and acquires the + same lock for every connection. So do in a strand because two threads + will ping-pong each other with cacheline invalidations. Revisit if + algorithm changes or the locking strategy within the levin config + class changes. */ + + std::vector<boost::uuids::uuid> connections; + connections.reserve(connection_id_reserve_size); + zone_->p2p->foreach_connection([this, &connections] (detail::p2p_context& context) { + if (this->source_ != context.m_connection_id) + connections.emplace_back(context.m_connection_id); + return true; + }); + + for (const boost::uuids::uuid& connection : connections) + zone_->p2p->send(message_.clone(), connection); + } + }; + + //! Updates the connection for a channel. + struct update_channel + { + std::shared_ptr<detail::zone> zone_; + const std::size_t channel_; + const boost::uuids::uuid connection_; + + //! \pre Called within `stem_.strand`. + void operator()() const + { + if (!zone_) + return; + + noise_channel& channel = zone_->channels.at(channel_); + assert(channel.strand.running_in_this_thread()); + static_assert( + CRYPTONOTE_MAX_FRAGMENTS <= (noise_min_epoch / (noise_min_delay + noise_delay_range)), + "Max fragments more than the max that can be sent in an epoch" + ); + + /* This clears the active message so that a message "in-flight" is + restarted. DO NOT try to send the remainder of the fragments, this + additional send time can leak that this node was sending out a real + notify (tx) instead of dummy noise. */ + + channel.connection = connection_; + channel.active = nullptr; + + if (connection_.is_nil()) + channel.queue.clear(); + } + }; + + //! Merges `out_connections_` into the existing `zone_->map`. + struct update_channels + { + std::shared_ptr<detail::zone> zone_; + std::vector<boost::uuids::uuid> out_connections_; + + //! \pre Called within `zone->strand`. + static void post(std::shared_ptr<detail::zone> zone) + { + if (!zone) + return; + + assert(zone->strand.running_in_this_thread()); + + zone->connection_count = zone->map.size(); + for (auto id = zone->map.begin(); id != zone->map.end(); ++id) + { + const std::size_t i = id - zone->map.begin(); + zone->channels[i].strand.post(update_channel{zone, i, *id}); + } + } + + //! \pre Called within `zone_->strand`. + void operator()() + { + if (!zone_) + return; + + assert(zone_->strand.running_in_this_thread()); + if (zone_->map.update(std::move(out_connections_))) + post(std::move(zone_)); + } + }; + + //! Swaps out noise channels entirely; new epoch start. + class change_channels + { + std::shared_ptr<detail::zone> zone_; + net::dandelionpp::connection_map map_; // Requires manual copy constructor + + public: + explicit change_channels(std::shared_ptr<detail::zone> zone, net::dandelionpp::connection_map map) + : zone_(std::move(zone)), map_(std::move(map)) + {} + + change_channels(change_channels&&) = default; + change_channels(const change_channels& source) + : zone_(source.zone_), map_(source.map_.clone()) + {} + + //! \pre Called within `zone_->strand`. + void operator()() + { + if (!zone_) + return + + assert(zone_->strand.running_in_this_thread()); + + zone_->map = std::move(map_); + update_channels::post(std::move(zone_)); + } + }; + + //! Sends a noise packet or real notification and sets timer for next call. + struct send_noise + { + std::shared_ptr<detail::zone> zone_; + const std::size_t channel_; + + static void wait(const std::chrono::steady_clock::time_point start, std::shared_ptr<detail::zone> zone, const std::size_t index) + { + if (!zone) + return; + + noise_channel& channel = zone->channels.at(index); + channel.next_noise.expires_at(start + noise_min_delay + random_duration(noise_delay_range)); + channel.next_noise.async_wait( + channel.strand.wrap(send_noise{std::move(zone), index}) + ); + } + + //! \pre Called within `zone_->channels[channel_].strand`. + void operator()(boost::system::error_code error) + { + if (!zone_ || !zone_->p2p || zone_->noise.empty()) + return; + + if (error && error != boost::system::errc::operation_canceled) + throw boost::system::system_error{error, "send_noise timer failed"}; + + assert(zone_->channels.at(channel_).strand.running_in_this_thread()); + + const auto start = std::chrono::steady_clock::now(); + noise_channel& channel = zone_->channels.at(channel_); + + if (!channel.connection.is_nil()) + { + epee::byte_slice message = nullptr; + if (!channel.active.empty()) + message = channel.active.take_slice(zone_->noise.size()); + else if (!channel.queue.empty()) + { + channel.active = channel.queue.front().clone(); + message = channel.active.take_slice(zone_->noise.size()); + } + else + message = zone_->noise.clone(); + + if (zone_->p2p->send(std::move(message), channel.connection)) + { + if (!channel.queue.empty() && channel.active.empty()) + channel.queue.pop_front(); + } + else + { + channel.active = nullptr; + channel.connection = boost::uuids::nil_uuid(); + zone_->strand.post( + update_channels{zone_, get_out_connections(*zone_->p2p)} + ); + } + } + + wait(start, std::move(zone_), channel_); + } + }; + + //! Prepares connections for new channel epoch and sets timer for next epoch + struct start_epoch + { + // Variables allow for Dandelion++ extension + std::shared_ptr<detail::zone> zone_; + std::chrono::seconds min_epoch_; + std::chrono::seconds epoch_range_; + std::size_t count_; + + //! \pre Should not be invoked within any strand to prevent blocking. + void operator()(const boost::system::error_code error = {}) + { + if (!zone_ || !zone_->p2p) + return; + + if (error && error != boost::system::errc::operation_canceled) + throw boost::system::system_error{error, "start_epoch timer failed"}; + + const auto start = std::chrono::steady_clock::now(); + zone_->strand.dispatch( + change_channels{zone_, net::dandelionpp::connection_map{get_out_connections(*(zone_->p2p)), count_}} + ); + + detail::zone& alias = *zone_; + alias.next_epoch.expires_at(start + min_epoch_ + random_duration(epoch_range_)); + alias.next_epoch.async_wait(start_epoch{std::move(*this)}); + } + }; + } // anonymous + + notify::notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise) + : zone_(std::make_shared<detail::zone>(service, std::move(p2p), std::move(noise))) + { + if (!zone_->p2p) + throw std::logic_error{"cryptonote::levin::notify cannot have nullptr p2p argument"}; + + if (!zone_->noise.empty()) + { + const auto now = std::chrono::steady_clock::now(); + start_epoch{zone_, noise_min_epoch, noise_epoch_range, CRYPTONOTE_NOISE_CHANNELS}(); + for (std::size_t channel = 0; channel < zone_->channels.size(); ++channel) + send_noise::wait(now, zone_, channel); + } + } + + notify::~notify() noexcept + {} + + notify::status notify::get_status() const noexcept + { + if (!zone_) + return {false, false}; + + return {!zone_->noise.empty(), CRYPTONOTE_NOISE_CHANNELS <= zone_->connection_count}; + } + + void notify::new_out_connection() + { + if (!zone_ || zone_->noise.empty() || CRYPTONOTE_NOISE_CHANNELS <= zone_->connection_count) + return; + + zone_->strand.dispatch( + update_channels{zone_, get_out_connections(*(zone_->p2p))} + ); + } + + void notify::run_epoch() + { + if (!zone_) + return; + zone_->next_epoch.cancel(); + } + + void notify::run_stems() + { + if (!zone_) + return; + + for (noise_channel& channel : zone_->channels) + channel.next_noise.cancel(); + } + + bool notify::send_txs(std::vector<cryptonote::blobdata> txs, const boost::uuids::uuid& source, const bool pad_txs) + { + if (!zone_) + return false; + + if (!zone_->noise.empty() && !zone_->channels.empty()) + { + // covert send in "noise" channel + static_assert( + CRYPTONOTE_MAX_FRAGMENTS * CRYPTONOTE_NOISE_BYTES <= LEVIN_DEFAULT_MAX_PACKET_SIZE, "most nodes will reject this fragment setting" + ); + + // padding is not useful when using noise mode + const std::string payload = make_tx_payload(std::move(txs), false); + epee::byte_slice message = epee::levin::make_fragmented_notify( + zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(payload) + ); + if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size()) + { + MERROR("notify::send_txs provided message exceeding covert fragment size"); + return false; + } + + for (std::size_t channel = 0; channel < zone_->channels.size(); ++channel) + { + zone_->channels[channel].strand.dispatch( + queue_covert_notify{zone_, message.clone(), channel} + ); + } + } + else + { + const std::string payload = make_tx_payload(std::move(txs), pad_txs); + epee::byte_slice message = + epee::levin::make_notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(payload)); + + // traditional monero send technique + zone_->strand.dispatch(flood_notify{zone_, std::move(message), source}); + } + + return true; + } +} // levin +} // net diff --git a/src/cryptonote_protocol/levin_notify.h b/src/cryptonote_protocol/levin_notify.h new file mode 100644 index 000000000..82d22680a --- /dev/null +++ b/src/cryptonote_protocol/levin_notify.h @@ -0,0 +1,132 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/asio/io_service.hpp> +#include <boost/uuid/uuid.hpp> +#include <memory> +#include <vector> + +#include "byte_slice.h" +#include "cryptonote_basic/blobdatatype.h" +#include "net/enums.h" +#include "span.h" + +namespace epee +{ +namespace levin +{ + template<typename> class async_protocol_handler_config; +} +} + +namespace nodetool +{ + template<typename> struct p2p_connection_context_t; +} + +namespace cryptonote +{ + struct cryptonote_connection_context; +} + +namespace cryptonote +{ +namespace levin +{ + namespace detail + { + using p2p_context = nodetool::p2p_connection_context_t<cryptonote::cryptonote_connection_context>; + struct zone; //!< Internal data needed for zone notifications + } // detail + + using connections = epee::levin::async_protocol_handler_config<detail::p2p_context>; + + //! Provides tx notification privacy + class notify + { + std::shared_ptr<detail::zone> zone_; + + public: + struct status + { + bool has_noise; + bool connections_filled; + }; + + //! Construct an instance that cannot notify. + notify() noexcept + : zone_(nullptr) + {} + + //! Construct an instance with available notification `zones`. + explicit notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise); + + notify(const notify&) = delete; + notify(notify&&) = default; + + ~notify() noexcept; + + notify& operator=(const notify&) = delete; + notify& operator=(notify&&) = default; + + //! \return Status information for zone selection. + status get_status() const noexcept; + + //! Probe for new outbound connection - skips if not needed. + void new_out_connection(); + + //! Run the logic for the next epoch immediately. Only use in testing. + void run_epoch(); + + //! Run the logic for the next stem timeout imemdiately. Only use in testing. + void run_stems(); + + /*! Send txs using `cryptonote_protocol_defs.h` payload format wrapped in a + levin header. The message will be sent in a "discreet" manner if + enabled - if `!noise.empty()` then the `command`/`payload` will be + queued to send at the next available noise interval. Otherwise, a + standard Monero flood notification will be used. + + \note Eventually Dandelion++ stem sending will be used here when + enabled. + + \param txs The transactions that need to be serialized and relayed. + \param source The source of the notification. `is_nil()` indicates this + node is the source. Dandelion++ will use this to map a source to a + particular stem. + \param pad_txs A request to pad txs to help conceal origin via + statistical analysis. Ignored if noise was enabled during + construction. + + \return True iff the notification is queued for sending. */ + bool send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source, bool pad_txs); + }; +} // levin +} // net diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 924447701..d47823735 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -51,12 +51,14 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& { if (args.size() > 3) { - std::cout << "use: print_pl [white] [gray] [<limit>]" << std::endl; + std::cout << "use: print_pl [white] [gray] [<limit>] [pruned] [publicrpc]" << std::endl; return true; } bool white = false; bool gray = false; + bool pruned = false; + bool publicrpc = false; size_t limit = 0; for (size_t i = 0; i < args.size(); ++i) { @@ -68,6 +70,14 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& { gray = true; } + else if (args[i] == "pruned") + { + pruned = true; + } + else if (args[i] == "publicrpc") + { + publicrpc = true; + } else if (!epee::string_tools::get_xtype_from_string(limit, args[i])) { std::cout << "unexpected argument: " << args[i] << std::endl; @@ -76,7 +86,7 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& } const bool print_both = !white && !gray; - return m_executor.print_peer_list(white | print_both, gray | print_both, limit); + return m_executor.print_peer_list(white | print_both, gray | print_both, limit, pruned, publicrpc); } bool t_command_parser_executor::print_peer_list_stats(const std::vector<std::string>& args) @@ -813,4 +823,18 @@ bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std:: return m_executor.check_blockchain_pruning(); } +bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::string>& args) +{ + const size_t args_count = args.size(); + if (args_count < 1 || args_count > 3) + { + return false; + } + + return m_executor.set_bootstrap_daemon( + args[0] != "none" ? args[0] : std::string(), + args_count > 1 ? args[1] : std::string(), + args_count > 2 ? args[2] : std::string()); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index d39bc1c9b..25587dea8 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -150,6 +150,8 @@ public: bool check_blockchain_pruning(const std::vector<std::string>& args); bool print_net_stats(const std::vector<std::string>& args); + + bool set_bootstrap_daemon(const std::vector<std::string>& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index aecdda52c..757e072a4 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -310,6 +310,13 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1) , "Check the blockchain pruning." ); + m_command_lookup.set_handler( + "set_bootstrap_daemon" + , std::bind(&t_command_parser_executor::set_bootstrap_daemon, &m_parser, p::_1) + , "set_bootstrap_daemon (auto | none | host[:port] [username] [password])" + , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced.\n" + "Use 'auto' to enable automatic public nodes discovering and bootstrap daemon switching" + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 461888062..cb288071e 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -312,9 +312,7 @@ int main(int argc, char const * argv[]) { login = tools::login::parse( has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), false, [](bool verify) { -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); return tools::password_container::prompt(verify, "Daemon client password"); } ); @@ -336,9 +334,7 @@ int main(int argc, char const * argv[]) } else { -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); std::cerr << "Unknown command: " << command.front() << std::endl; return 1; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 4d3debed6..d4b9434da 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -60,13 +60,18 @@ namespace { } } - void print_peer(std::string const & prefix, cryptonote::peer const & peer) + void print_peer(std::string const & prefix, cryptonote::peer const & peer, bool pruned_only, bool publicrpc_only) { + if (pruned_only && peer.pruning_seed == 0) + return; + if (publicrpc_only && peer.rpc_port == 0) + return; + time_t now; time(&now); time_t last_seen = static_cast<time_t>(peer.last_seen); - std::string elapsed = epee::misc_utils::get_time_interval_string(now - last_seen); + std::string elapsed = peer.last_seen == 0 ? "never" : epee::misc_utils::get_time_interval_string(now - last_seen); std::string id_str = epee::string_tools::pad_string(epee::string_tools::to_string_hex(peer.id), 16, '0', true); std::string port_str; epee::string_tools::xtype_to_string(peer.port, port_str); @@ -169,7 +174,7 @@ t_rpc_command_executor::~t_rpc_command_executor() } } -bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit) { +bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit, bool pruned_only, bool publicrpc_only) { cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; @@ -196,7 +201,7 @@ bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit const auto end = limit ? peer + std::min(limit, res.white_list.size()) : res.white_list.cend(); for (; peer != end; ++peer) { - print_peer("white", *peer); + print_peer("white", *peer, pruned_only, publicrpc_only); } } @@ -206,7 +211,7 @@ bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit const auto end = limit ? peer + std::min(limit, res.gray_list.size()) : res.gray_list.cend(); for (; peer != end; ++peer) { - print_peer("gray", *peer); + print_peer("gray", *peer, pruned_only, publicrpc_only); } } @@ -2320,4 +2325,40 @@ bool t_rpc_command_executor::check_blockchain_pruning() return true; } +bool t_rpc_command_executor::set_bootstrap_daemon( + const std::string &address, + const std::string &username, + const std::string &password) +{ + cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::request req; + cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::response res; + const std::string fail_message = "Unsuccessful"; + + req.address = address; + req.username = username; + req.password = password; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_bootstrap_daemon", fail_message)) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_bootstrap_daemon(req, res) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + tools::success_msg_writer() + << "Successfully set bootstrap daemon address to " + << (!req.address.empty() ? req.address : "none"); + + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index f3ed48319..af7081ef3 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -69,7 +69,7 @@ public: ~t_rpc_command_executor(); - bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0); + bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0, bool pruned_only = false, bool publicrpc_only = false); bool print_peer_list_stats(); @@ -162,6 +162,11 @@ public: bool check_blockchain_pruning(); bool print_net_stats(); + + bool set_bootstrap_daemon( + const std::string &address, + const std::string &username, + const std::string &password); }; } // namespace daemonize diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index eba633da8..15fe560d1 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -320,7 +320,9 @@ namespace hw { bool device_ledger::reset() { reset_buffer(); int offset = set_command_header_noopt(INS_RESET); - memmove(this->buffer_send+offset, MONERO_VERSION, strlen(MONERO_VERSION)); + const size_t verlen = strlen(MONERO_VERSION); + ASSERT_X(offset + verlen <= BUFFER_SEND_SIZE, "MONERO_VERSION is too long") + memmove(this->buffer_send+offset, MONERO_VERSION, verlen); offset += strlen(MONERO_VERSION); this->buffer_send[4] = offset-5; this->length_send = offset; @@ -416,10 +418,10 @@ namespace hw { #ifdef DEBUG_HWDEVICE cryptonote::account_public_address pubkey; this->get_public_address(pubkey); - #endif crypto::secret_key vkey; crypto::secret_key skey; this->get_secret_keys(vkey,skey); + #endif return true; } diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index fe9028733..986087128 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -76,7 +76,7 @@ namespace hw { rct::key AKout; ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, bool is_subaddress, bool is_change_address, size_t index, const rct::key& P,const rct::key& AK); ABPkeys(const ABPkeys& keys) ; - ABPkeys() {index=0;is_subaddress=false;is_subaddress=false;is_change_address=false;} + ABPkeys() {index=0;is_subaddress=false;is_change_address=false;additional_key=false;} ABPkeys &operator=(const ABPkeys &keys); }; diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 738f858f0..24b707f77 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,8 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources error.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp) -set(net_headers error.h i2p_address.h parse.h socks.h socks_connect.h tor_address.h) +set(net_sources dandelionpp.cpp error.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp) +set(net_headers dandelionpp.h error.h i2p_address.h parse.h socks.h socks_connect.h tor_address.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${Boost_ASIO_LIBRARY}) diff --git a/src/net/dandelionpp.cpp b/src/net/dandelionpp.cpp new file mode 100644 index 000000000..4d2f75428 --- /dev/null +++ b/src/net/dandelionpp.cpp @@ -0,0 +1,212 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "dandelionpp.h" + +#include <boost/container/small_vector.hpp> +#include <boost/uuid/nil_generator.hpp> +#include <chrono> + +#include "common/expect.h" +#include "cryptonote_config.h" +#include "crypto/crypto.h" + +namespace net +{ +namespace dandelionpp +{ + namespace + { + constexpr const std::size_t expected_max_channels = CRYPTONOTE_NOISE_CHANNELS; + + // could be in util somewhere + struct key_less + { + template<typename K, typename V> + bool operator()(const std::pair<K, V>& left, const K& right) const + { + return left.first < right; + } + + template<typename K, typename V> + bool operator()(const K& left, const std::pair<K, V>& right) const + { + return left < right.first; + } + }; + + std::size_t select_stem(epee::span<const std::size_t> usage, epee::span<const boost::uuids::uuid> out_map) + { + assert(usage.size() < std::numeric_limits<std::size_t>::max()); // prevented in constructor + if (usage.size() < out_map.size()) + return std::numeric_limits<std::size_t>::max(); + + // small_vector uses stack space if `expected_max_channels < capacity()` + std::size_t lowest = std::numeric_limits<std::size_t>::max(); + boost::container::small_vector<std::size_t, expected_max_channels> choices; + static_assert(sizeof(choices) < 256, "choices is too large based on current configuration"); + + for (const boost::uuids::uuid& out : out_map) + { + if (!out.is_nil()) + { + const std::size_t location = std::addressof(out) - out_map.begin(); + if (usage[location] < lowest) + { + lowest = usage[location]; + choices = {location}; + } + else if (usage[location] == lowest) + choices.push_back(location); + } + } + + switch (choices.size()) + { + case 0: + return std::numeric_limits<std::size_t>::max(); + case 1: + return choices[0]; + default: + break; + } + + return choices[crypto::rand_idx(choices.size())]; + } + } // anonymous + + connection_map::connection_map(std::vector<boost::uuids::uuid> out_connections, const std::size_t stems) + : out_mapping_(std::move(out_connections)), + in_mapping_(), + usage_count_() + { + // max value is used by `select_stem` as error case + if (stems == std::numeric_limits<std::size_t>::max()) + MONERO_THROW(common_error::kInvalidArgument, "stems value cannot be max size_t"); + + usage_count_.resize(stems); + if (stems < out_mapping_.size()) + { + for (unsigned i = 0; i < stems; ++i) + std::swap(out_mapping_[i], out_mapping_.at(i + crypto::rand_idx(out_mapping_.size() - i))); + + out_mapping_.resize(stems); + } + else + { + std::shuffle(out_mapping_.begin(), out_mapping_.end(), crypto::random_device{}); + } + } + + connection_map::~connection_map() noexcept + {} + + connection_map connection_map::clone() const + { + return {*this}; + } + + bool connection_map::update(std::vector<boost::uuids::uuid> current) + { + std::sort(current.begin(), current.end()); + + bool replace = false; + for (auto& existing_out : out_mapping_) + { + const auto elem = std::lower_bound(current.begin(), current.end(), existing_out); + if (elem == current.end() || *elem != existing_out) + { + existing_out = boost::uuids::nil_uuid(); + replace = true; + } + else // already using connection, remove it from candidate list + current.erase(elem); + } + + if (!replace && out_mapping_.size() == usage_count_.size()) + return false; + + const std::size_t existing_outs = out_mapping_.size(); + for (std::size_t i = 0; i < usage_count_.size() && !current.empty(); ++i) + { + const bool increase_stems = out_mapping_.size() <= i; + if (increase_stems || out_mapping_[i].is_nil()) + { + std::swap(current.back(), current.at(crypto::rand_idx(current.size()))); + if (increase_stems) + out_mapping_.push_back(current.back()); + else + out_mapping_[i] = current.back(); + current.pop_back(); + } + } + + return replace || existing_outs < out_mapping_.size(); + } + + std::size_t connection_map::size() const noexcept + { + std::size_t count = 0; + for (const boost::uuids::uuid& connection : out_mapping_) + { + if (!connection.is_nil()) + ++count; + } + return count; + } + + boost::uuids::uuid connection_map::get_stem(const boost::uuids::uuid& source) + { + auto elem = std::lower_bound(in_mapping_.begin(), in_mapping_.end(), source, key_less{}); + if (elem == in_mapping_.end() || elem->first != source) + { + const std::size_t index = select_stem(epee::to_span(usage_count_), epee::to_span(out_mapping_)); + if (out_mapping_.size() < index) + return boost::uuids::nil_uuid(); + + elem = in_mapping_.emplace(elem, source, index); + usage_count_[index]++; + } + else if (out_mapping_.at(elem->second).is_nil()) // stem connection disconnected after mapping + { + usage_count_.at(elem->second)--; + const std::size_t index = select_stem(epee::to_span(usage_count_), epee::to_span(out_mapping_)); + if (out_mapping_.size() < index) + { + in_mapping_.erase(elem); + return boost::uuids::nil_uuid(); + } + + elem->second = index; + usage_count_[index]++; + } + + return out_mapping_[elem->second]; + } +} // dandelionpp +} // net diff --git a/src/net/dandelionpp.h b/src/net/dandelionpp.h new file mode 100644 index 000000000..75b63bc0c --- /dev/null +++ b/src/net/dandelionpp.h @@ -0,0 +1,106 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/uuid/uuid.hpp> +#include <cstddef> +#include <memory> +#include <utility> +#include <vector> + +#include "span.h" + +namespace net +{ +namespace dandelionpp +{ + //! Assists with mapping source -> stem and tracking connections for stem. + class connection_map + { + // Make sure to update clone method if changing members + std::vector<boost::uuids::uuid> out_mapping_; //<! Current outgoing uuid connection at index. + std::vector<std::pair<boost::uuids::uuid, std::size_t>> in_mapping_; //<! uuid source to an `out_mapping_` index. + std::vector<std::size_t> usage_count_; + + // Use clone method to prevent "hidden" copies. + connection_map(const connection_map&) = default; + + public: + using value_type = boost::uuids::uuid; + using size_type = std::vector<boost::uuids::uuid>::size_type; + using difference_type = std::vector<boost::uuids::uuid>::difference_type; + using reference = const boost::uuids::uuid&; + using const_reference = reference; + using iterator = std::vector<boost::uuids::uuid>::const_iterator; + using const_iterator = iterator; + + //! Initialized with zero stem connections. + explicit connection_map() + : connection_map(std::vector<boost::uuids::uuid>{}, 0) + {} + + //! Initialized with `out_connections` and `stem_count`. + explicit connection_map(std::vector<boost::uuids::uuid> out_connections, std::size_t stems); + + connection_map(connection_map&&) = default; + ~connection_map() noexcept; + connection_map& operator=(connection_map&&) = default; + connection_map& operator=(const connection_map&) = delete; + + //! \return An exact duplicate of `this` map. + connection_map clone() const; + + //! \return First stem connection. + const_iterator begin() const noexcept + { + return out_mapping_.begin(); + } + + //! \return One-past the last stem connection. + const_iterator end() const noexcept + { + return out_mapping_.end(); + } + + /*! Merges in current connections with the previous set of connections. + If a connection died, a new one will take its place in the stem or + the stem is marked as dead. + + \param connections Current outbound connection ids. + \return True if any updates to `get_connections()` was made. */ + bool update(std::vector<boost::uuids::uuid> current); + + //! \return Number of outgoing connections in use. + std::size_t size() const noexcept; + + //! \return Current stem mapping for `source` or `nil_uuid()` if none is possible. + boost::uuids::uuid get_stem(const boost::uuids::uuid& source); + }; +} // dandelionpp +} // net diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index bb51be242..c7fc058ca 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -144,7 +144,7 @@ namespace nodetool const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_add_exclusive_node = {"add-exclusive-node", "Specify list of peers to connect to only." " If this option is given the options add-priority-node and seed-node are ignored"}; const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; - const command_line::arg_descriptor<std::vector<std::string> > arg_proxy = {"proxy", "<network-type>,<socks-ip:port>[,max_connections] i.e. \"tor,127.0.0.1:9050,100\""}; + const command_line::arg_descriptor<std::vector<std::string> > arg_proxy = {"proxy", "<network-type>,<socks-ip:port>[,max_connections][,disable_noise] i.e. \"tor,127.0.0.1:9050,100,disable_noise\""}; const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound = {"anonymous-inbound", "<hidden-service-address>,<[bind-ip:]port>[,max_connections] i.e. \"x.onion,127.0.0.1:18083,100\""}; const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; const command_line::arg_descriptor<bool> arg_no_sync = {"no-sync", "Don't synchronize the blockchain with other peers", false}; @@ -163,7 +163,7 @@ namespace nodetool boost::optional<std::vector<proxy>> get_proxies(boost::program_options::variables_map const& vm) { - namespace ip = boost::asio::ip; + namespace ip = boost::asio::ip; std::vector<proxy> proxies{}; @@ -183,14 +183,25 @@ namespace nodetool const boost::string_ref proxy{next->begin(), next->size()}; ++next; - if (!next.eof()) + for (unsigned count = 0; !next.eof(); ++count, ++next) { - proxies.back().max_connections = get_max_connections(*next); - if (proxies.back().max_connections == 0) + if (2 <= count) { - MERROR("Invalid max connections given to --" << arg_proxy.name); + MERROR("Too many ',' characters given to --" << arg_proxy.name); return boost::none; } + + if (boost::string_ref{next->begin(), next->size()} == "disable_noise") + proxies.back().noise = false; + else + { + proxies.back().max_connections = get_max_connections(*next); + if (proxies.back().max_connections == 0) + { + MERROR("Invalid max connections given to --" << arg_proxy.name); + return boost::none; + } + } } switch (epee::net_utils::zone_from_string(zone)) @@ -214,7 +225,7 @@ namespace nodetool return boost::none; } proxies.back().address = ip::tcp::endpoint{ip::address_v4{boost::endian::native_to_big(ip)}, port}; - } + } return proxies; } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 6d2ae878f..255a1fc1f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -43,6 +43,7 @@ #include <vector> #include "cryptonote_config.h" +#include "cryptonote_protocol/levin_notify.h" #include "warnings.h" #include "net/abstract_tcp_server2.h" #include "net/levin_protocol_handler.h" @@ -66,12 +67,14 @@ namespace nodetool proxy() : max_connections(-1), address(), - zone(epee::net_utils::zone::invalid) + zone(epee::net_utils::zone::invalid), + noise(true) {} std::int64_t max_connections; boost::asio::ip::tcp::endpoint address; epee::net_utils::zone zone; + bool noise; }; struct anonymous_inbound @@ -154,6 +157,7 @@ namespace nodetool m_bind_ipv6_address(), m_port(), m_port_ipv6(), + m_notifier(), m_our_address(), m_peerlist(), m_config{}, @@ -172,6 +176,7 @@ namespace nodetool m_bind_ipv6_address(), m_port(), m_port_ipv6(), + m_notifier(), m_our_address(), m_peerlist(), m_config{}, @@ -189,6 +194,7 @@ namespace nodetool std::string m_bind_ipv6_address; std::string m_port; std::string m_port_ipv6; + cryptonote::levin::notify m_notifier; epee::net_utils::network_address m_our_address; // in anonymity networks peerlist_manager m_peerlist; config m_config; @@ -255,7 +261,6 @@ namespace nodetool size_t get_public_gray_peers_count(); void get_public_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white); void get_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white); - size_t get_zone_count() const { return m_network_zones.size(); } void change_max_out_public_peers(size_t count); uint32_t get_max_out_public_peers() const; @@ -330,6 +335,7 @@ namespace nodetool virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs); virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); @@ -349,8 +355,7 @@ namespace nodetool bool get_local_node_data(basic_node_data& node_data, const network_zone& zone); //bool get_local_handshake_data(handshake_data& hshd); - bool merge_peerlist_with_local(const std::vector<peerlist_entry>& bs); - bool fix_time_delta(std::vector<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta); + bool sanitize_peerlist(std::vector<peerlist_entry>& local_peerlist); bool connections_maker(); bool peer_sync_idle_maker(); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 8c0cff7e2..97a18b519 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -383,6 +383,9 @@ namespace nodetool m_offline = command_line::get_arg(vm, cryptonote::arg_offline); m_use_ipv6 = command_line::get_arg(vm, arg_p2p_use_ipv6); m_require_ipv4 = command_line::get_arg(vm, arg_p2p_require_ipv4); + public_zone.m_notifier = cryptonote::levin::notify{ + public_zone.m_net_server.get_io_service(), public_zone.m_net_server.get_config_shared(), nullptr + }; if (command_line::has_arg(vm, arg_p2p_add_peer)) { @@ -462,6 +465,7 @@ namespace nodetool return false; + epee::byte_slice noise = nullptr; auto proxies = get_proxies(vm); if (!proxies) return false; @@ -479,6 +483,20 @@ namespace nodetool if (!set_max_out_peers(zone, proxy.max_connections)) return false; + + epee::byte_slice this_noise = nullptr; + if (proxy.noise) + { + static_assert(sizeof(epee::levin::bucket_head2) < CRYPTONOTE_NOISE_BYTES, "noise bytes too small"); + if (noise.empty()) + noise = epee::levin::make_noise_notify(CRYPTONOTE_NOISE_BYTES); + + this_noise = noise.clone(); + } + + zone.m_notifier = cryptonote::levin::notify{ + zone.m_net_server.get_io_service(), zone.m_net_server.get_config_shared(), std::move(this_noise) + }; } for (const auto& zone : m_network_zones) @@ -494,6 +512,7 @@ namespace nodetool if (!inbounds) return false; + const std::size_t tx_relay_zones = m_network_zones.size(); for (auto& inbound : *inbounds) { network_zone& zone = add_zone(inbound.our_address.get_zone()); @@ -504,6 +523,12 @@ namespace nodetool return false; } + if (zone.m_connect == nullptr && tx_relay_zones <= 1) + { + MERROR("Listed --" << arg_anonymous_inbound.name << " without listing any --" << arg_proxy.name << ". The latter is necessary for sending origin txes over anonymity networks"); + return false; + } + zone.m_bind_ip = std::move(inbound.local_ip); zone.m_port = std::move(inbound.local_port); zone.m_net_server.set_default_remote(std::move(inbound.default_remote)); @@ -1266,6 +1291,7 @@ namespace nodetool ape.first_seen = first_seen_stamp ? first_seen_stamp : time(nullptr); zone.m_peerlist.append_with_peer_anchor(ape); + zone.m_notifier.new_out_connection(); LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK."); return true; @@ -1420,7 +1446,7 @@ namespace nodetool if (skipped == 0 || !filtered.empty()) break; if (skipped) - MGINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); + MINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); } if (filtered.empty()) { @@ -1815,21 +1841,32 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::fix_time_delta(std::vector<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta) + bool node_server<t_payload_net_handler>::sanitize_peerlist(std::vector<peerlist_entry>& local_peerlist) { - //fix time delta - time_t now = 0; - time(&now); - delta = now - local_time; - - for(peerlist_entry& be: local_peerlist) + for (size_t i = 0; i < local_peerlist.size(); ++i) { - if(be.last_seen > local_time) + bool ignore = false; + peerlist_entry &be = local_peerlist[i]; + epee::net_utils::network_address &na = be.adr; + if (na.is_loopback() || na.is_local()) { - MWARNING("FOUND FUTURE peerlist for entry " << be.adr.str() << " last_seen: " << be.last_seen << ", local_time(on remote node):" << local_time); - return false; + ignore = true; } - be.last_seen += delta; + else if (be.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); + if (ipv4.ip() == 0) + ignore = true; + } + if (ignore) + { + MDEBUG("Ignoring " << be.adr.str()); + std::swap(local_peerlist[i], local_peerlist[local_peerlist.size() - 1]); + local_peerlist.resize(local_peerlist.size() - 1); + --i; + continue; + } + #ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); #endif @@ -1840,9 +1877,8 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::handle_remote_peerlist(const std::vector<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context) { - int64_t delta = 0; std::vector<peerlist_entry> peerlist_ = peerlist; - if(!fix_time_delta(peerlist_, local_time, delta)) + if(!sanitize_peerlist(peerlist_)) return false; const epee::net_utils::zone zone = context.m_remote_address.get_zone(); @@ -1855,8 +1891,8 @@ namespace nodetool } } - LOG_DEBUG_CC(context, "REMOTE PEERLIST: TIME_DELTA: " << delta << ", remote peerlist size=" << peerlist_.size()); - LOG_DEBUG_CC(context, "REMOTE PEERLIST: " << print_peerlist_to_string(peerlist_)); + LOG_DEBUG_CC(context, "REMOTE PEERLIST: remote peerlist size=" << peerlist_.size()); + LOG_DEBUG_CC(context, "REMOTE PEERLIST: " << ENDL << print_peerlist_to_string(peerlist_)); return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_); } //----------------------------------------------------------------------------------- @@ -1990,7 +2026,7 @@ namespace nodetool } if (c_id.first <= zone->first) break; - + ++zone; } if (zone->first == c_id.first) @@ -2000,6 +2036,61 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> + epee::net_utils::zone node_server<t_payload_net_handler>::send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs) + { + namespace enet = epee::net_utils; + + const auto send = [&txs, &source, pad_txs] (std::pair<const enet::zone, network_zone>& network) + { + if (network.second.m_notifier.send_txs(std::move(txs), source, (pad_txs || network.first != enet::zone::public_))) + return network.first; + return enet::zone::invalid; + }; + + if (m_network_zones.empty()) + return enet::zone::invalid; + + if (origin != enet::zone::invalid) + return send(*m_network_zones.begin()); // send all txs received via p2p over public network + + if (m_network_zones.size() <= 2) + return send(*m_network_zones.rbegin()); // see static asserts below; sends over anonymity network iff enabled + + /* These checks are to ensure that i2p is highest priority if multiple + zones are selected. Make sure to update logic if the values cannot be + in the same relative order. `m_network_zones` must be sorted map too. */ + static_assert(std::is_same<std::underlying_type<enet::zone>::type, std::uint8_t>{}, "expected uint8_t zone"); + static_assert(unsigned(enet::zone::invalid) == 0, "invalid expected to be 0"); + static_assert(unsigned(enet::zone::public_) == 1, "public_ expected to be 1"); + static_assert(unsigned(enet::zone::i2p) == 2, "i2p expected to be 2"); + static_assert(unsigned(enet::zone::tor) == 3, "tor expected to be 3"); + + // check for anonymity networks with noise and connections + for (auto network = ++m_network_zones.begin(); network != m_network_zones.end(); ++network) + { + if (enet::zone::tor < network->first) + break; // unknown network + + const auto status = network->second.m_notifier.get_status(); + if (status.has_noise && status.connections_filled) + return send(*network); + } + + // use the anonymity network with outbound support + for (auto network = ++m_network_zones.begin(); network != m_network_zones.end(); ++network) + { + if (enet::zone::tor < network->first) + break; // unknown network + + if (network->second.m_connect) + return send(*network); + } + + // configuration should not allow this scenario + return enet::zone::invalid; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> void node_server<t_payload_net_handler>::callback(p2p_connection_context& context) { m_payload_handler.on_callback(context); @@ -2227,6 +2318,15 @@ namespace nodetool network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + // test only the remote end's zone, otherwise an attacker could connect to you on clearnet + // and pass in a tor connection's peer id, and deduce the two are the same if you reject it + if(arg.node_data.peer_id == zone.m_config.m_peer_id) + { + LOG_DEBUG_CC(context, "Connection to self detected, dropping connection"); + drop_connection(context); + return 1; + } + if (zone.m_current_number_of_in_peers >= zone.m_config.m_net_config.max_in_connection_count) // in peers limit { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came, but already have max incoming connections, so dropping this one."); @@ -2253,7 +2353,7 @@ namespace nodetool context.m_in_timedsync = false; context.m_rpc_port = arg.node_data.rpc_port; - if(arg.node_data.peer_id != zone.m_config.m_peer_id && arg.node_data.my_port && zone.m_can_pingback) + if(arg.node_data.my_port && zone.m_can_pingback) { peerid_type peer_id_l = arg.node_data.peer_id; uint32_t port_l = arg.node_data.my_port; diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 34d151f5f..239814c2c 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -33,6 +33,8 @@ #include <boost/uuid/uuid.hpp> #include <utility> #include <vector> +#include "cryptonote_basic/blobdatatype.h" +#include "net/enums.h" #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" @@ -46,12 +48,12 @@ namespace nodetool struct i_p2p_endpoint { virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs)=0; virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_public_connections_count()=0; - virtual size_t get_zone_count() const=0; virtual void for_each_connection(std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0; virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0; virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds = 0)=0; @@ -71,6 +73,10 @@ namespace nodetool { return false; } + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, const bool pad_txs) + { + return epee::net_utils::zone::invalid; + } virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { return false; @@ -96,11 +102,6 @@ namespace nodetool return false; } - virtual size_t get_zone_count() const - { - return 0; - } - virtual uint64_t get_public_connections_count() { return false; diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 05eb36e65..c2773981c 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -95,7 +95,9 @@ namespace boost { uint32_t ip{na.ip()}; uint16_t port{na.port()}; + ip = SWAP32LE(ip); a & ip; + ip = SWAP32LE(ip); a & port; if (!typename Archive::is_saving()) na = epee::net_utils::ipv4_network_address{ip, port}; diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index 6270d4d14..ff6fee95c 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -101,7 +101,10 @@ static rct::key get_exponent(const rct::key &base, size_t idx) { static const std::string salt("bulletproof"); std::string hashed = std::string((const char*)base.bytes, sizeof(base)) + salt + tools::get_varint_data(idx); - const rct::key e = rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + rct::key e; + ge_p3 e_p3; + rct::hash_to_p3(e_p3, rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + ge_p3_tobytes(e.bytes, &e_p3); CHECK_AND_ASSERT_THROW_MES(!(e == rct::identity()), "Exponent is point at infinity"); return e; } diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index b5499262f..6e4d063df 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -620,44 +620,16 @@ namespace rct { sc_reduce32(rv.bytes); return rv; } - - key hashToPointSimple(const key & hh) { - key pointk; - ge_p1p1 point2; - ge_p2 point; - ge_p3 res; - key h = cn_fast_hash(hh); - CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&res, h.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); - ge_p3_to_p2(&point, &res); - ge_mul8(&point2, &point); - ge_p1p1_to_p3(&res, &point2); - ge_p3_tobytes(pointk.bytes, &res); - return pointk; - } - key hashToPoint(const key & hh) { - key pointk; - ge_p2 point; - ge_p1p1 point2; - ge_p3 res; - key h = cn_fast_hash(hh); - ge_fromfe_frombytes_vartime(&point, h.bytes); - ge_mul8(&point2, &point); - ge_p1p1_to_p3(&res, &point2); - ge_p3_tobytes(pointk.bytes, &res); - return pointk; - } - - void hashToPoint(key & pointk, const key & hh) { - ge_p2 point; - ge_p1p1 point2; - ge_p3 res; - key h = cn_fast_hash(hh); - ge_fromfe_frombytes_vartime(&point, h.bytes); - ge_mul8(&point2, &point); - ge_p1p1_to_p3(&res, &point2); - ge_p3_tobytes(pointk.bytes, &res); - } + // Hash a key to p3 representation + void hash_to_p3(ge_p3 &hash8_p3, const key &k) { + key hash_key = cn_fast_hash(k); + ge_p2 hash_p2; + ge_fromfe_frombytes_vartime(&hash_p2, hash_key.bytes); + ge_p1p1 hash8_p1p1; + ge_mul8(&hash8_p1p1, &hash_p2); + ge_p1p1_to_p3(&hash8_p3, &hash8_p1p1); + } //sums a vector of curve points (for scalars use sc_add) void sumKeys(key & Csum, const keyV & Cis) { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index dd6d87593..c24d48e9a 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -172,10 +172,7 @@ namespace rct { key cn_fast_hash(const key64 keys); key hash_to_scalar(const key64 keys); - //returns hashToPoint as described in https://github.com/ShenNoether/ge_fromfe_writeup - key hashToPointSimple(const key &in); - key hashToPoint(const key &in); - void hashToPoint(key &out, const key &in); + void hash_to_p3(ge_p3 &hash8_p3, const key &k); //sums a vector of curve points (for scalars use sc_add) void sumKeys(key & Csum, const key &Cis); diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index ff2a81d43..a7b265d63 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -163,14 +163,11 @@ namespace rct { return verifyBorromean(bb, P1_p3, P2_p3); } - //Multilayered Spontaneous Anonymous Group Signatures (MLSAG signatures) - //This is a just slghtly more efficient version than the ones described below - //(will be explained in more detail in Ring Multisig paper - //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. https://eprint.iacr.org/2015/1098 section 2. - // Gen creates a signature which proves that for some column in the keymatrix "pk" - // the signer knows a secret key for each row in that column - // Ver verifies that the MG sig was created correctly + // MLSAG signatures + // See paper by Noether (https://eprint.iacr.org/2015/1098) + // This generalization allows for some dimensions not to require linkability; + // this is used in practice for commitment data within signatures + // Note that using more than one linkable dimension is not recommended. mgSig MLSAG_Gen(const key &message, const keyM & pk, const keyV & xx, const multisig_kLRki *kLRki, key *mscout, const unsigned int index, size_t dsRows, hw::device &hwdev) { mgSig rv; size_t cols = pk.size(); @@ -188,6 +185,7 @@ namespace rct { size_t i = 0, j = 0, ii = 0; key c, c_old, L, R, Hi; + ge_p3 Hi_p3; sc_0(c_old.bytes); vector<geDsmp> Ip(dsRows); rv.II = keyV(dsRows); @@ -208,7 +206,8 @@ namespace rct { rv.II[i] = kLRki->ki; } else { - Hi = hashToPoint(pk[index][i]); + hash_to_p3(Hi_p3, pk[index][i]); + ge_p3_tobytes(Hi.bytes, &Hi_p3); hwdev.mlsag_prepare(Hi, xx[i], alpha[i] , aG[i] , aHP[i] , rv.II[i]); toHash[3 * i + 2] = aG[i]; toHash[3 * i + 3] = aHP[i]; @@ -235,7 +234,8 @@ namespace rct { sc_0(c.bytes); for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); - hashToPoint(Hi, pk[i][j]); + hash_to_p3(Hi_p3, pk[i][j]); + ge_p3_tobytes(Hi.bytes, &Hi_p3); addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; @@ -260,43 +260,42 @@ namespace rct { return rv; } - //Multilayered Spontaneous Anonymous Group Signatures (MLSAG signatures) - //This is a just slghtly more efficient version than the ones described below - //(will be explained in more detail in Ring Multisig paper - //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. https://eprint.iacr.org/2015/1098 section 2. - // Gen creates a signature which proves that for some column in the keymatrix "pk" - // the signer knows a secret key for each row in that column - // Ver verifies that the MG sig was created correctly + // MLSAG signatures + // See paper by Noether (https://eprint.iacr.org/2015/1098) + // This generalization allows for some dimensions not to require linkability; + // this is used in practice for commitment data within signatures + // Note that using more than one linkable dimension is not recommended. bool MLSAG_Ver(const key &message, const keyM & pk, const mgSig & rv, size_t dsRows) { - size_t cols = pk.size(); - CHECK_AND_ASSERT_MES(cols >= 2, false, "Error! What is c if cols = 1!"); + CHECK_AND_ASSERT_MES(cols >= 2, false, "Signature must contain more than one public key"); size_t rows = pk[0].size(); - CHECK_AND_ASSERT_MES(rows >= 1, false, "Empty pk"); + CHECK_AND_ASSERT_MES(rows >= 1, false, "Bad total row number"); for (size_t i = 1; i < cols; ++i) { - CHECK_AND_ASSERT_MES(pk[i].size() == rows, false, "pk is not rectangular"); + CHECK_AND_ASSERT_MES(pk[i].size() == rows, false, "Bad public key matrix dimensions"); } - CHECK_AND_ASSERT_MES(rv.II.size() == dsRows, false, "Bad II size"); - CHECK_AND_ASSERT_MES(rv.ss.size() == cols, false, "Bad rv.ss size"); + CHECK_AND_ASSERT_MES(rv.II.size() == dsRows, false, "Wrong number of key images present"); + CHECK_AND_ASSERT_MES(rv.ss.size() == cols, false, "Bad scalar matrix dimensions"); for (size_t i = 0; i < cols; ++i) { - CHECK_AND_ASSERT_MES(rv.ss[i].size() == rows, false, "rv.ss is not rectangular"); + CHECK_AND_ASSERT_MES(rv.ss[i].size() == rows, false, "Bad scalar matrix dimensions"); } - CHECK_AND_ASSERT_MES(dsRows <= rows, false, "Bad dsRows value"); + CHECK_AND_ASSERT_MES(dsRows <= rows, false, "Non-double-spend rows cannot exceed total rows"); - for (size_t i = 0; i < rv.ss.size(); ++i) - for (size_t j = 0; j < rv.ss[i].size(); ++j) - CHECK_AND_ASSERT_MES(sc_check(rv.ss[i][j].bytes) == 0, false, "Bad ss slot"); - CHECK_AND_ASSERT_MES(sc_check(rv.cc.bytes) == 0, false, "Bad cc"); + for (size_t i = 0; i < rv.ss.size(); ++i) { + for (size_t j = 0; j < rv.ss[i].size(); ++j) { + CHECK_AND_ASSERT_MES(sc_check(rv.ss[i][j].bytes) == 0, false, "Bad signature scalar"); + } + } + CHECK_AND_ASSERT_MES(sc_check(rv.cc.bytes) == 0, false, "Bad initial signature hash"); size_t i = 0, j = 0, ii = 0; - key c, L, R, Hi; + key c, L, R; key c_old = copy(rv.cc); vector<geDsmp> Ip(dsRows); for (i = 0 ; i < dsRows ; i++) { + CHECK_AND_ASSERT_MES(!(rv.II[i] == rct::identity()), false, "Bad key image"); precomp(Ip[i].k, rv.II[i]); } - size_t ndsRows = 3 * dsRows; //non Double Spendable Rows (see identity chains paper + size_t ndsRows = 3 * dsRows; // number of dimensions not requiring linkability keyV toHash(1 + 3 * dsRows + 2 * (rows - dsRows)); toHash[0] = message; i = 0; @@ -304,9 +303,14 @@ namespace rct { sc_0(c.bytes); for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); - hashToPoint(Hi, pk[i][j]); - CHECK_AND_ASSERT_MES(!(Hi == rct::identity()), false, "Data hashed to point at infinity"); - addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); + + // Compute R directly + ge_p3 hash8_p3; + hash_to_p3(hash8_p3, pk[i][j]); + ge_p2 R_p2; + ge_double_scalarmult_precomp_vartime(&R_p2, rv.ss[i][j].bytes, &hash8_p3, c_old.bytes, Ip[j].k); + ge_tobytes(R.bytes, &R_p2); + toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; toHash[3 * j + 3] = R; @@ -317,6 +321,7 @@ namespace rct { toHash[ndsRows + 2 * ii + 2] = L; } c = hash_to_scalar(toHash); + CHECK_AND_ASSERT_MES(!(c == rct::zero()), false, "Bad signature hash"); copy(c_old, c); i = (i + 1); } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 06577d37e..d98670e05 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -30,6 +30,7 @@ set(rpc_base_sources rpc_args.cpp) set(rpc_sources + bootstrap_daemon.cpp core_rpc_server.cpp rpc_handler.cpp instanciations) @@ -53,6 +54,7 @@ set(daemon_rpc_server_headers) set(rpc_daemon_private_headers + bootstrap_daemon.h core_rpc_server.h core_rpc_server_commands_defs.h core_rpc_server_error_codes.h) diff --git a/src/rpc/bootstrap_daemon.cpp b/src/rpc/bootstrap_daemon.cpp new file mode 100644 index 000000000..6f8426ee7 --- /dev/null +++ b/src/rpc/bootstrap_daemon.cpp @@ -0,0 +1,95 @@ +#include "bootstrap_daemon.h" + +#include <stdexcept> + +#include "crypto/crypto.h" +#include "cryptonote_core/cryptonote_core.h" +#include "misc_log_ex.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc.bootstrap_daemon" + +namespace cryptonote +{ + + bootstrap_daemon::bootstrap_daemon(std::function<boost::optional<std::string>()> get_next_public_node) noexcept + : m_get_next_public_node(get_next_public_node) + { + } + + bootstrap_daemon::bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials) + : bootstrap_daemon(nullptr) + { + if (!set_server(address, credentials)) + { + throw std::runtime_error("invalid bootstrap daemon address or credentials"); + } + } + + std::string bootstrap_daemon::address() const noexcept + { + const auto& host = m_http_client.get_host(); + if (host.empty()) + { + return std::string(); + } + return host + ":" + m_http_client.get_port(); + } + + boost::optional<uint64_t> bootstrap_daemon::get_height() + { + cryptonote::COMMAND_RPC_GET_HEIGHT::request req; + cryptonote::COMMAND_RPC_GET_HEIGHT::response res; + + if (!invoke_http_json("/getheight", req, res)) + { + return boost::none; + } + + if (res.status != CORE_RPC_STATUS_OK) + { + return boost::none; + } + + return res.height; + } + + bool bootstrap_daemon::handle_result(bool success) + { + if (!success && m_get_next_public_node) + { + m_http_client.disconnect(); + } + + return success; + } + + bool bootstrap_daemon::set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials /* = boost::none */) + { + if (!m_http_client.set_server(address, credentials)) + { + MERROR("Failed to set bootstrap daemon address " << address); + return false; + } + + MINFO("Changed bootstrap daemon address to " << address); + return true; + } + + + bool bootstrap_daemon::switch_server_if_needed() + { + if (!m_get_next_public_node || m_http_client.is_connected()) + { + return true; + } + + const boost::optional<std::string> address = m_get_next_public_node(); + if (address) { + return set_server(*address); + } + + return false; + } + +} diff --git a/src/rpc/bootstrap_daemon.h b/src/rpc/bootstrap_daemon.h new file mode 100644 index 000000000..130a6458d --- /dev/null +++ b/src/rpc/bootstrap_daemon.h @@ -0,0 +1,67 @@ +#pragma once + +#include <functional> +#include <vector> + +#include <boost/optional/optional.hpp> +#include <boost/utility/string_ref.hpp> + +#include "net/http_client.h" +#include "storages/http_abstract_invoke.h" + +namespace cryptonote +{ + + class bootstrap_daemon + { + public: + bootstrap_daemon(std::function<boost::optional<std::string>()> get_next_public_node) noexcept; + bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials); + + std::string address() const noexcept; + boost::optional<uint64_t> get_height(); + bool handle_result(bool success); + + template <class t_request, class t_response> + bool invoke_http_json(const boost::string_ref uri, const t_request &out_struct, t_response &result_struct) + { + if (!switch_server_if_needed()) + { + return false; + } + + return handle_result(epee::net_utils::invoke_http_json(uri, out_struct, result_struct, m_http_client)); + } + + template <class t_request, class t_response> + bool invoke_http_bin(const boost::string_ref uri, const t_request &out_struct, t_response &result_struct) + { + if (!switch_server_if_needed()) + { + return false; + } + + return handle_result(epee::net_utils::invoke_http_bin(uri, out_struct, result_struct, m_http_client)); + } + + template <class t_request, class t_response> + bool invoke_http_json_rpc(const boost::string_ref command_name, const t_request &out_struct, t_response &result_struct) + { + if (!switch_server_if_needed()) + { + return false; + } + + return handle_result(epee::net_utils::invoke_http_json_rpc("/json_rpc", std::string(command_name.begin(), command_name.end()), out_struct, result_struct, m_http_client)); + } + + private: + bool set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials = boost::none); + bool switch_server_if_needed(); + + private: + epee::net_utils::http::http_simple_client m_http_client; + std::function<boost::optional<std::string>()> m_get_next_public_node; + }; + +} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 73138686d..7706d2cf7 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -103,6 +103,7 @@ namespace cryptonote ) : m_core(cr) , m_p2p(p2p) + , m_was_bootstrap_ever_used(false) {} //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const std::string &username_password) @@ -116,20 +117,59 @@ namespace cryptonote return set_bootstrap_daemon(address, credentials); } //------------------------------------------------------------------------------------------------------------------------------ + boost::optional<std::string> core_rpc_server::get_random_public_node() + { + COMMAND_RPC_GET_PUBLIC_NODES::request request; + COMMAND_RPC_GET_PUBLIC_NODES::response response; + + request.gray = true; + request.white = true; + if (!on_get_public_nodes(request, response) || response.status != CORE_RPC_STATUS_OK) + { + return boost::none; + } + + const auto get_random_node_address = [](const std::vector<public_node>& public_nodes) -> std::string { + const auto& random_node = public_nodes[crypto::rand_idx(public_nodes.size())]; + const auto address = random_node.host + ":" + std::to_string(random_node.rpc_port); + return address; + }; + + if (!response.white.empty()) + { + return get_random_node_address(response.white); + } + + MDEBUG("No white public node found, checking gray peers"); + + if (!response.gray.empty()) + { + return get_random_node_address(response.gray); + } + + MERROR("Failed to find any suitable public node"); + + return boost::none; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials) { boost::unique_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); - if (!address.empty()) + if (address.empty()) { - if (!m_http_client.set_server(address, credentials, epee::net_utils::ssl_support_t::e_ssl_support_autodetect)) - { - return false; - } + m_bootstrap_daemon.reset(nullptr); + } + else if (address == "auto") + { + m_bootstrap_daemon.reset(new bootstrap_daemon([this]{ return get_random_public_node(); })); + } + else + { + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials)); } - m_bootstrap_daemon_address = address; - m_should_use_bootstrap_daemon = !m_bootstrap_daemon_address.empty(); + m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; return true; } @@ -220,7 +260,10 @@ namespace cryptonote { { boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); - res.bootstrap_daemon_address = m_bootstrap_daemon_address; + if (m_bootstrap_daemon.get() != nullptr) + { + res.bootstrap_daemon_address = m_bootstrap_daemon->address(); + } } crypto::hash top_hash; m_core.get_blockchain_top(res.height_without_bootstrap, top_hash); @@ -269,7 +312,10 @@ namespace cryptonote else { boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); - res.bootstrap_daemon_address = m_bootstrap_daemon_address; + if (m_bootstrap_daemon.get() != nullptr) + { + res.bootstrap_daemon_address = m_bootstrap_daemon->address(); + } res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); @@ -1593,8 +1639,10 @@ namespace cryptonote boost::upgrade_lock<boost::shared_mutex> upgrade_lock(m_bootstrap_daemon_mutex); - if (m_bootstrap_daemon_address.empty()) + if (m_bootstrap_daemon.get() == nullptr) + { return false; + } if (!m_should_use_bootstrap_daemon) { @@ -1610,42 +1658,38 @@ namespace cryptonote m_bootstrap_height_check_time = current_time; } - uint64_t top_height; - crypto::hash top_hash; - m_core.get_blockchain_top(top_height, top_hash); - ++top_height; // turn top block height into blockchain height + boost::optional<uint64_t> bootstrap_daemon_height = m_bootstrap_daemon->get_height(); + if (!bootstrap_daemon_height) + { + MERROR("Failed to fetch bootstrap daemon height"); + return false; + } - // query bootstrap daemon's height - cryptonote::COMMAND_RPC_GET_HEIGHT::request getheight_req; - cryptonote::COMMAND_RPC_GET_HEIGHT::response getheight_res; - bool ok = epee::net_utils::invoke_http_json("/getheight", getheight_req, getheight_res, m_http_client); - ok = ok && getheight_res.status == CORE_RPC_STATUS_OK; + uint64_t target_height = m_core.get_target_blockchain_height(); + if (*bootstrap_daemon_height < target_height) + { + MINFO("Bootstrap daemon is out of sync"); + return m_bootstrap_daemon->handle_result(false); + } - m_should_use_bootstrap_daemon = ok && top_height + 10 < getheight_res.height; - MINFO((m_should_use_bootstrap_daemon ? "Using" : "Not using") << " the bootstrap daemon (our height: " << top_height << ", bootstrap daemon's height: " << (ok ? getheight_res.height : 0) << ")"); + uint64_t top_height = m_core.get_current_blockchain_height(); + m_should_use_bootstrap_daemon = top_height + 10 < *bootstrap_daemon_height; + MINFO((m_should_use_bootstrap_daemon ? "Using" : "Not using") << " the bootstrap daemon (our height: " << top_height << ", bootstrap daemon's height: " << *bootstrap_daemon_height << ")"); } if (!m_should_use_bootstrap_daemon) return false; if (mode == invoke_http_mode::JON) { - r = epee::net_utils::invoke_http_json(command_name, req, res, m_http_client); + r = m_bootstrap_daemon->invoke_http_json(command_name, req, res); } else if (mode == invoke_http_mode::BIN) { - r = epee::net_utils::invoke_http_bin(command_name, req, res, m_http_client); + r = m_bootstrap_daemon->invoke_http_bin(command_name, req, res); } else if (mode == invoke_http_mode::JON_RPC) { - epee::json_rpc::request<typename COMMAND_TYPE::request> json_req = AUTO_VAL_INIT(json_req); - epee::json_rpc::response<typename COMMAND_TYPE::response, std::string> json_resp = AUTO_VAL_INIT(json_resp); - json_req.jsonrpc = "2.0"; - json_req.id = epee::serialization::storage_entry(0); - json_req.method = command_name; - json_req.params = req; - r = net_utils::invoke_http_json("/json_rpc", json_req, json_resp, m_http_client); - if (r) - res = json_resp.result; + r = m_bootstrap_daemon->invoke_http_json_rpc(command_name, req, res); } else { @@ -2147,6 +2191,7 @@ namespace cryptonote return r; res.version = CORE_RPC_VERSION; + res.release = MONERO_VERSION_IS_RELEASE; res.status = CORE_RPC_STATUS_OK; return true; } @@ -2617,7 +2662,8 @@ namespace cryptonote const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_address = { "bootstrap-daemon-address" - , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced" + , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced.\n" + "Use 'auto' to enable automatic public nodes discovering and bootstrap daemon switching" , "" }; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index e91d4c953..379f6ed28 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -30,9 +30,12 @@ #pragma once +#include <memory> + #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> +#include "bootstrap_daemon.h" #include "net/http_server_impl_base.h" #include "net/http_client.h" #include "core_rpc_server_commands_defs.h" @@ -243,6 +246,7 @@ private: //utils uint64_t get_block_reward(const block& blk); bool fill_block_header_response(const block& blk, bool orphan_status, uint64_t height, const crypto::hash& hash, block_header_response& response, bool fill_pow_hash); + boost::optional<std::string> get_random_public_node(); bool set_bootstrap_daemon(const std::string &address, const std::string &username_password); bool set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials); enum invoke_http_mode { JON, BIN, JON_RPC }; @@ -251,9 +255,8 @@ private: core& m_core; nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_p2p; - std::string m_bootstrap_daemon_address; - epee::net_utils::http::http_simple_client m_http_client; boost::shared_mutex m_bootstrap_daemon_mutex; + std::unique_ptr<bootstrap_daemon> m_bootstrap_daemon; bool m_should_use_bootstrap_daemon; std::chrono::system_clock::time_point m_bootstrap_height_check_time; bool m_was_bootstrap_ever_used; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index aed967efb..325ac4343 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -87,7 +87,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 9 +#define CORE_RPC_VERSION_MINOR 10 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -2053,11 +2053,13 @@ namespace cryptonote { std::string status; uint32_t version; + bool release; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(version) + KV_SERIALIZE(release) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 5d7321e48..eee2fa61d 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -33,6 +33,7 @@ * * \brief Source file that defines simple_wallet class. */ +#include <locale.h> #include <thread> #include <iostream> #include <sstream> @@ -45,6 +46,7 @@ #include <boost/regex.hpp> #include <boost/range/adaptor/transformed.hpp> #include "include_base_utils.h" +#include "console_handler.h" #include "common/i18n.h" #include "common/command_line.h" #include "common/util.h" @@ -245,6 +247,7 @@ namespace const char* USAGE_FREEZE("freeze <key_image>"); const char* USAGE_THAW("thaw <key_image>"); const char* USAGE_FROZEN("frozen <key_image>"); + const char* USAGE_LOCK("lock"); const char* USAGE_NET_STATS("net_stats"); const char* USAGE_WELCOME("welcome"); const char* USAGE_VERSION("version"); @@ -252,9 +255,7 @@ namespace std::string input_line(const std::string& prompt, bool yesno = false) { -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); std::cout << prompt; if (yesno) std::cout << " (Y/Yes/N/No)"; @@ -272,9 +273,7 @@ namespace epee::wipeable_string input_secure_line(const char *prompt) { -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); auto pwd_container = tools::password_container::prompt(false, prompt, false); if (!pwd_container) { @@ -290,9 +289,7 @@ namespace boost::optional<tools::password_container> password_prompter(const char *prompt, bool verify) { -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); auto pwd_container = tools::password_container::prompt(verify, prompt); if (!pwd_container) { @@ -1254,7 +1251,7 @@ bool simple_wallet::export_multisig_main(const std::vector<std::string> &args, b } else { - bool r = epee::file_io_utils::save_string_to_file(filename, ciphertext); + bool r = m_wallet->save_to_file(filename, ciphertext); if (!r) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -1315,7 +1312,7 @@ bool simple_wallet::import_multisig_main(const std::vector<std::string> &args, b { const std::string &filename = args[n]; std::string data; - bool r = epee::file_io_utils::load_file_to_string(filename, data); + bool r = m_wallet->load_from_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to read file ") << filename; @@ -1626,7 +1623,7 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) if (!filenames.empty()) filenames += ", "; filenames += filename; - if (!epee::file_io_utils::save_string_to_file(filename, cryptonote::tx_to_blob(ptx.tx))) + if (!m_wallet->save_to_file(filename, cryptonote::tx_to_blob(ptx.tx))) { fail_msg_writer() << tr("Failed to export multisig transaction to file ") << filename; return true; @@ -2141,6 +2138,13 @@ bool simple_wallet::frozen(const std::vector<std::string> &args) return true; } +bool simple_wallet::lock(const std::vector<std::string> &args) +{ + m_locked = true; + check_for_inactivity_lock(true); + return true; +} + bool simple_wallet::net_stats(const std::vector<std::string> &args) { message_writer() << std::to_string(m_wallet->get_bytes_sent()) + tr(" bytes sent"); @@ -2172,6 +2176,25 @@ bool simple_wallet::version(const std::vector<std::string> &args) return true; } +bool simple_wallet::on_unknown_command(const std::vector<std::string> &args) +{ + if (args[0] == "exit" || args[0] == "q") // backward compat + return false; + fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help'")) % args.front(); + return true; +} + +bool simple_wallet::on_empty_command() +{ + return true; +} + +bool simple_wallet::on_cancelled_command() +{ + check_for_inactivity_lock(false); + return true; +} + bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func) { std::vector<std::string> tx_aux; @@ -2660,6 +2683,29 @@ bool simple_wallet::set_track_uses(const std::vector<std::string> &args/* = std: return true; } +bool simple_wallet::set_inactivity_lock_timeout(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ +#ifdef _WIN32 + tools::fail_msg_writer() << tr("Inactivity lock timeout disabled on Windows"); + return true; +#endif + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + uint32_t r; + if (epee::string_tools::get_xtype_from_string(r, args[1])) + { + m_wallet->inactivity_lock_timeout(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + } + else + { + tools::fail_msg_writer() << tr("Invalid number of seconds"); + } + } + return true; +} + bool simple_wallet::set_setup_background_mining(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { const auto pwd_container = get_and_verify_password(); @@ -2712,6 +2758,35 @@ bool simple_wallet::set_device_name(const std::vector<std::string> &args/* = std return true; } +bool simple_wallet::set_export_format(const std::vector<std::string> &args/* = std::vector<std::string()*/) +{ + if (args.size() < 2) + { + fail_msg_writer() << tr("Export format not specified"); + return true; + } + + if (boost::algorithm::iequals(args[1], "ascii")) + { + m_wallet->set_export_format(tools::wallet2::ExportFormat::Ascii); + } + else if (boost::algorithm::iequals(args[1], "binary")) + { + m_wallet->set_export_format(tools::wallet2::ExportFormat::Binary); + } + else + { + fail_msg_writer() << tr("Export format not recognized."); + return true; + } + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) @@ -2739,83 +2814,86 @@ simple_wallet::simple_wallet() , m_auto_refresh_refreshing(false) , m_in_manual_refresh(false) , m_current_subaddress_account(0) + , m_last_activity_time(time(NULL)) + , m_locked(false) + , m_in_command(false) { m_cmd_binder.set_handler("start_mining", - boost::bind(&simple_wallet::start_mining, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::start_mining, _1), tr(USAGE_START_MINING), tr("Start mining in the daemon (bg_mining and ignore_battery are optional booleans).")); m_cmd_binder.set_handler("stop_mining", - boost::bind(&simple_wallet::stop_mining, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::stop_mining, _1), tr("Stop mining in the daemon.")); m_cmd_binder.set_handler("set_daemon", - boost::bind(&simple_wallet::set_daemon, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_daemon, _1), tr(USAGE_SET_DAEMON), tr("Set another daemon to connect to.")); m_cmd_binder.set_handler("save_bc", - boost::bind(&simple_wallet::save_bc, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::save_bc, _1), tr("Save the current blockchain data.")); m_cmd_binder.set_handler("refresh", - boost::bind(&simple_wallet::refresh, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::refresh, _1), tr("Synchronize the transactions and balance.")); m_cmd_binder.set_handler("balance", - boost::bind(&simple_wallet::show_balance, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_balance, _1), tr(USAGE_SHOW_BALANCE), tr("Show the wallet's balance of the currently selected account.")); m_cmd_binder.set_handler("incoming_transfers", - boost::bind(&simple_wallet::show_incoming_transfers, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_incoming_transfers,_1), tr(USAGE_INCOMING_TRANSFERS), tr("Show the incoming transfers, all or filtered by availability and address index.\n\n" "Output format:\n" "Amount, Spent(\"T\"|\"F\"), \"frozen\"|\"locked\"|\"unlocked\", RingCT, Global Index, Transaction Hash, Address Index, [Public Key, Key Image] ")); m_cmd_binder.set_handler("payments", - boost::bind(&simple_wallet::show_payments, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_payments,_1), tr(USAGE_PAYMENTS), tr("Show the payments for the given payment IDs.")); m_cmd_binder.set_handler("bc_height", - boost::bind(&simple_wallet::show_blockchain_height, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_blockchain_height, _1), tr("Show the blockchain height.")); - m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), + m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::transfer, _1), tr(USAGE_TRANSFER), tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", - boost::bind(&simple_wallet::locked_transfer, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::locked_transfer,_1), tr(USAGE_LOCKED_TRANSFER), tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_sweep_all", - boost::bind(&simple_wallet::locked_sweep_all, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::locked_sweep_all,_1), tr(USAGE_LOCKED_SWEEP_ALL), tr("Send all unlocked balance to an address and lock it for <lockblocks> (max. 1000000). If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. <priority> is the priority of the sweep. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability.")); m_cmd_binder.set_handler("sweep_unmixable", - boost::bind(&simple_wallet::sweep_unmixable, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::sweep_unmixable, _1), tr("Send all unmixable outputs to yourself with ring_size 1")); m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), tr(USAGE_SWEEP_ALL), tr("Send all unlocked balance to an address. If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and N > 0, wallet splits the transaction into N even outputs.")); m_cmd_binder.set_handler("sweep_below", - boost::bind(&simple_wallet::sweep_below, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::sweep_below, _1), tr(USAGE_SWEEP_BELOW), tr("Send all unlocked outputs below the threshold to an address.")); m_cmd_binder.set_handler("sweep_single", - boost::bind(&simple_wallet::sweep_single, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::sweep_single, _1), tr(USAGE_SWEEP_SINGLE), tr("Send a single output of the given key image to an address without change.")); m_cmd_binder.set_handler("donate", - boost::bind(&simple_wallet::donate, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::donate, _1), tr(USAGE_DONATE), tr("Donate <amount> to the development team (donate.getmonero.org).")); m_cmd_binder.set_handler("sign_transfer", - boost::bind(&simple_wallet::sign_transfer, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign_transfer, _1), tr(USAGE_SIGN_TRANSFER), tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); m_cmd_binder.set_handler("submit_transfer", - boost::bind(&simple_wallet::submit_transfer, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::submit_transfer, _1), tr("Submit a signed transaction from a file.")); m_cmd_binder.set_handler("set_log", - boost::bind(&simple_wallet::set_log, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_log, _1), tr(USAGE_SET_LOG), tr("Change the current log detail (level must be <0-4>).")); m_cmd_binder.set_handler("account", - boost::bind(&simple_wallet::account, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::account, _1), tr(USAGE_ACCOUNT), tr("If no arguments are specified, the wallet shows all the existing accounts along with their balances.\n" "If the \"new\" argument is specified, the wallet creates a new account with its label initialized by the provided label text (which can be empty).\n" @@ -2825,37 +2903,37 @@ simple_wallet::simple_wallet() "If the \"untag\" argument is specified, the tags assigned to the specified accounts <account_index_1>, <account_index_2> ..., are removed.\n" "If the \"tag_description\" argument is specified, the tag <tag_name> is assigned an arbitrary text <description>.")); m_cmd_binder.set_handler("address", - boost::bind(&simple_wallet::print_address, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::print_address, _1), tr(USAGE_ADDRESS), tr("If no arguments are specified or <index> is specified, the wallet shows the default or specified address. If \"all\" is specified, the wallet shows all the existing addresses in the currently selected account. If \"new \" is specified, the wallet creates a new address with the provided label text (which can be empty). If \"label\" is specified, the wallet sets the label of the address specified by <index> to the provided label text.")); m_cmd_binder.set_handler("integrated_address", - boost::bind(&simple_wallet::print_integrated_address, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::print_integrated_address, _1), tr(USAGE_INTEGRATED_ADDRESS), tr("Encode a payment ID into an integrated address for the current wallet public address (no argument uses a random payment ID), or decode an integrated address to standard address and payment ID")); m_cmd_binder.set_handler("address_book", - boost::bind(&simple_wallet::address_book, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::address_book,_1), tr(USAGE_ADDRESS_BOOK), tr("Print all entries in the address book, optionally adding/deleting an entry to/from it.")); m_cmd_binder.set_handler("save", - boost::bind(&simple_wallet::save, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::save, _1), tr("Save the wallet data.")); m_cmd_binder.set_handler("save_watch_only", - boost::bind(&simple_wallet::save_watch_only, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::save_watch_only, _1), tr("Save a watch-only keys file.")); m_cmd_binder.set_handler("viewkey", - boost::bind(&simple_wallet::viewkey, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::viewkey, _1), tr("Display the private view key.")); m_cmd_binder.set_handler("spendkey", - boost::bind(&simple_wallet::spendkey, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::spendkey, _1), tr("Display the private spend key.")); m_cmd_binder.set_handler("seed", - boost::bind(&simple_wallet::seed, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::seed, _1), tr("Display the Electrum-style mnemonic seed")); m_cmd_binder.set_handler("restore_height", boost::bind(&simple_wallet::restore_height, this, _1), tr("Display the restore height")); m_cmd_binder.set_handler("set", - boost::bind(&simple_wallet::set_variable, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_variable, _1), tr(USAGE_SET_VARIABLE), tr("Available options:\n " "seed language\n " @@ -2912,53 +2990,55 @@ simple_wallet::simple_wallet() "setup-background-mining <1|0>\n " " Whether to enable background mining. Set this to support the network and to get a chance to receive new monero.\n " "device-name <device_name[:device_spec]>\n " - " Device name for hardware wallet.")); + " Device name for hardware wallet.\n " + "export-format <\"binary\"|\"ascii\">\n " + " Save all exported files as binary (cannot be copied and pasted) or ascii (can be).\n ")); m_cmd_binder.set_handler("encrypted_seed", - boost::bind(&simple_wallet::encrypted_seed, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::encrypted_seed, _1), tr("Display the encrypted Electrum-style mnemonic seed.")); m_cmd_binder.set_handler("rescan_spent", - boost::bind(&simple_wallet::rescan_spent, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::rescan_spent, _1), tr("Rescan the blockchain for spent outputs.")); m_cmd_binder.set_handler("get_tx_key", - boost::bind(&simple_wallet::get_tx_key, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::get_tx_key, _1), tr(USAGE_GET_TX_KEY), tr("Get the transaction key (r) for a given <txid>.")); m_cmd_binder.set_handler("set_tx_key", - boost::bind(&simple_wallet::set_tx_key, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_tx_key, _1), tr(USAGE_SET_TX_KEY), tr("Set the transaction key (r) for a given <txid> in case the tx was made by some other device or 3rd party wallet.")); m_cmd_binder.set_handler("check_tx_key", - boost::bind(&simple_wallet::check_tx_key, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::check_tx_key, _1), tr(USAGE_CHECK_TX_KEY), tr("Check the amount going to <address> in <txid>.")); m_cmd_binder.set_handler("get_tx_proof", - boost::bind(&simple_wallet::get_tx_proof, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::get_tx_proof, _1), tr(USAGE_GET_TX_PROOF), tr("Generate a signature proving funds sent to <address> in <txid>, optionally with a challenge string <message>, using either the transaction secret key (when <address> is not your wallet's address) or the view secret key (otherwise), which does not disclose the secret key.")); m_cmd_binder.set_handler("check_tx_proof", - boost::bind(&simple_wallet::check_tx_proof, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::check_tx_proof, _1), tr(USAGE_CHECK_TX_PROOF), tr("Check the proof for funds going to <address> in <txid> with the challenge string <message> if any.")); m_cmd_binder.set_handler("get_spend_proof", - boost::bind(&simple_wallet::get_spend_proof, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::get_spend_proof, _1), tr(USAGE_GET_SPEND_PROOF), tr("Generate a signature proving that you generated <txid> using the spend secret key, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("check_spend_proof", - boost::bind(&simple_wallet::check_spend_proof, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::check_spend_proof, _1), tr(USAGE_CHECK_SPEND_PROOF), tr("Check a signature proving that the signer generated <txid>, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("get_reserve_proof", - boost::bind(&simple_wallet::get_reserve_proof, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::get_reserve_proof, _1), tr(USAGE_GET_RESERVE_PROOF), tr("Generate a signature proving that you own at least this much, optionally with a challenge string <message>.\n" "If 'all' is specified, you prove the entire sum of all of your existing accounts' balances.\n" "Otherwise, you prove the reserve of the smallest possible amount above <amount> available in your current account.")); m_cmd_binder.set_handler("check_reserve_proof", - boost::bind(&simple_wallet::check_reserve_proof, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::check_reserve_proof, _1), tr(USAGE_CHECK_RESERVE_PROOF), tr("Check a signature proving that the owner of <address> holds at least this much, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("show_transfers", - boost::bind(&simple_wallet::show_transfers, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_transfers, _1), tr(USAGE_SHOW_TRANSFERS), // Seemingly broken formatting to compensate for the backslash before the quotes. tr("Show the incoming/outgoing transfers within an optional height range.\n\n" @@ -2970,120 +3050,120 @@ simple_wallet::simple_wallet() "* Excluding change and fee.\n" "** Set of address indices used as inputs in this transfer.")); m_cmd_binder.set_handler("export_transfers", - boost::bind(&simple_wallet::export_transfers, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_transfers, _1), tr("export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"), tr("Export to CSV the incoming/outgoing transfers within an optional height range.")); m_cmd_binder.set_handler("unspent_outputs", - boost::bind(&simple_wallet::unspent_outputs, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::unspent_outputs, _1), tr(USAGE_UNSPENT_OUTPUTS), tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", - boost::bind(&simple_wallet::rescan_blockchain, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::rescan_blockchain, _1), tr(USAGE_RESCAN_BC), tr("Rescan the blockchain from scratch. If \"hard\" is specified, you will lose any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", - boost::bind(&simple_wallet::set_tx_note, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_tx_note, _1), tr(USAGE_SET_TX_NOTE), tr("Set an arbitrary string note for a <txid>.")); m_cmd_binder.set_handler("get_tx_note", - boost::bind(&simple_wallet::get_tx_note, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::get_tx_note, _1), tr(USAGE_GET_TX_NOTE), tr("Get a string note for a txid.")); m_cmd_binder.set_handler("set_description", - boost::bind(&simple_wallet::set_description, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_description, _1), tr(USAGE_SET_DESCRIPTION), tr("Set an arbitrary description for the wallet.")); m_cmd_binder.set_handler("get_description", - boost::bind(&simple_wallet::get_description, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::get_description, _1), tr(USAGE_GET_DESCRIPTION), tr("Get the description of the wallet.")); m_cmd_binder.set_handler("status", - boost::bind(&simple_wallet::status, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::status, _1), tr("Show the wallet's status.")); m_cmd_binder.set_handler("wallet_info", - boost::bind(&simple_wallet::wallet_info, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::wallet_info, _1), tr("Show the wallet's information.")); m_cmd_binder.set_handler("sign", - boost::bind(&simple_wallet::sign, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign, _1), tr(USAGE_SIGN), tr("Sign the contents of a file.")); m_cmd_binder.set_handler("verify", - boost::bind(&simple_wallet::verify, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::verify, _1), tr(USAGE_VERIFY), tr("Verify a signature on the contents of a file.")); m_cmd_binder.set_handler("export_key_images", - boost::bind(&simple_wallet::export_key_images, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_key_images, _1), tr(USAGE_EXPORT_KEY_IMAGES), tr("Export a signed set of key images to a <filename>.")); m_cmd_binder.set_handler("import_key_images", - boost::bind(&simple_wallet::import_key_images, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::import_key_images, _1), tr(USAGE_IMPORT_KEY_IMAGES), tr("Import a signed key images list and verify their spent status.")); m_cmd_binder.set_handler("hw_key_images_sync", - boost::bind(&simple_wallet::hw_key_images_sync, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::hw_key_images_sync, _1), tr(USAGE_HW_KEY_IMAGES_SYNC), tr("Synchronizes key images with the hw wallet.")); m_cmd_binder.set_handler("hw_reconnect", - boost::bind(&simple_wallet::hw_reconnect, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::hw_reconnect, _1), tr(USAGE_HW_RECONNECT), tr("Attempts to reconnect HW wallet.")); m_cmd_binder.set_handler("export_outputs", - boost::bind(&simple_wallet::export_outputs, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_outputs, _1), tr(USAGE_EXPORT_OUTPUTS), tr("Export a set of outputs owned by this wallet.")); m_cmd_binder.set_handler("import_outputs", - boost::bind(&simple_wallet::import_outputs, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::import_outputs, _1), tr(USAGE_IMPORT_OUTPUTS), tr("Import a set of outputs owned by this wallet.")); m_cmd_binder.set_handler("show_transfer", - boost::bind(&simple_wallet::show_transfer, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_transfer, _1), tr(USAGE_SHOW_TRANSFER), tr("Show information about a transfer to/from this address.")); m_cmd_binder.set_handler("password", - boost::bind(&simple_wallet::change_password, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::change_password, _1), tr("Change the wallet's password.")); m_cmd_binder.set_handler("payment_id", - boost::bind(&simple_wallet::payment_id, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::payment_id, _1), tr(USAGE_PAYMENT_ID), tr("Generate a new random full size payment id (obsolete). These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); m_cmd_binder.set_handler("fee", - boost::bind(&simple_wallet::print_fee_info, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::print_fee_info, _1), tr("Print the information about the current fee and transaction backlog.")); - m_cmd_binder.set_handler("prepare_multisig", boost::bind(&simple_wallet::prepare_multisig, this, _1), + m_cmd_binder.set_handler("prepare_multisig", boost::bind(&simple_wallet::on_command, this, &simple_wallet::prepare_multisig, _1), tr("Export data needed to create a multisig wallet")); - m_cmd_binder.set_handler("make_multisig", boost::bind(&simple_wallet::make_multisig, this, _1), + m_cmd_binder.set_handler("make_multisig", boost::bind(&simple_wallet::on_command, this, &simple_wallet::make_multisig, _1), tr(USAGE_MAKE_MULTISIG), tr("Turn this wallet into a multisig wallet")); m_cmd_binder.set_handler("finalize_multisig", - boost::bind(&simple_wallet::finalize_multisig, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::finalize_multisig, _1), tr(USAGE_FINALIZE_MULTISIG), tr("Turn this wallet into a multisig wallet, extra step for N-1/N wallets")); m_cmd_binder.set_handler("exchange_multisig_keys", - boost::bind(&simple_wallet::exchange_multisig_keys, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::exchange_multisig_keys, _1), tr(USAGE_EXCHANGE_MULTISIG_KEYS), tr("Performs extra multisig keys exchange rounds. Needed for arbitrary M/N multisig wallets")); m_cmd_binder.set_handler("export_multisig_info", - boost::bind(&simple_wallet::export_multisig, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_multisig, _1), tr(USAGE_EXPORT_MULTISIG_INFO), tr("Export multisig info for other participants")); m_cmd_binder.set_handler("import_multisig_info", - boost::bind(&simple_wallet::import_multisig, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::import_multisig, _1), tr(USAGE_IMPORT_MULTISIG_INFO), tr("Import multisig info from other participants")); m_cmd_binder.set_handler("sign_multisig", - boost::bind(&simple_wallet::sign_multisig, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign_multisig, _1), tr(USAGE_SIGN_MULTISIG), tr("Sign a multisig transaction from a file")); m_cmd_binder.set_handler("submit_multisig", - boost::bind(&simple_wallet::submit_multisig, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::submit_multisig, _1), tr(USAGE_SUBMIT_MULTISIG), tr("Submit a signed multisig transaction from a file")); m_cmd_binder.set_handler("export_raw_multisig_tx", - boost::bind(&simple_wallet::export_raw_multisig, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_raw_multisig, _1), tr(USAGE_EXPORT_RAW_MULTISIG_TX), tr("Export a signed multisig transaction to a file")); m_cmd_binder.set_handler("mms", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS), tr("Interface with the MMS (Multisig Messaging System)\n" "<subcommand> is one of:\n" @@ -3091,60 +3171,60 @@ simple_wallet::simple_wallet() " send_signer_config, start_auto_config, stop_auto_config, auto_config\n" "Get help about a subcommand with: help mms <subcommand>, or mms help <subcommand>")); m_cmd_binder.set_handler("mms init", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_INIT), tr("Initialize and configure the MMS for M/N = number of required signers/number of authorized signers multisig")); m_cmd_binder.set_handler("mms info", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_INFO), tr("Display current MMS configuration")); m_cmd_binder.set_handler("mms signer", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_SIGNER), tr("Set or modify authorized signer info (single-word label, transport address, Monero address), or list all signers")); m_cmd_binder.set_handler("mms list", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_LIST), tr("List all messages")); m_cmd_binder.set_handler("mms next", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_NEXT), tr("Evaluate the next possible multisig-related action(s) according to wallet state, and execute or offer for choice\n" "By using 'sync' processing of waiting messages with multisig sync info can be forced regardless of wallet state")); m_cmd_binder.set_handler("mms sync", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_SYNC), tr("Force generation of multisig sync info regardless of wallet state, to recover from special situations like \"stale data\" errors")); m_cmd_binder.set_handler("mms transfer", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_TRANSFER), tr("Initiate transfer with MMS support; arguments identical to normal 'transfer' command arguments, for info see there")); m_cmd_binder.set_handler("mms delete", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_DELETE), tr("Delete a single message by giving its id, or delete all messages by using 'all'")); m_cmd_binder.set_handler("mms send", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_SEND), tr("Send a single message by giving its id, or send all waiting messages")); m_cmd_binder.set_handler("mms receive", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_RECEIVE), tr("Check right away for new messages to receive")); m_cmd_binder.set_handler("mms export", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_EXPORT), tr("Write the content of a message to a file \"mms_message_content\"")); m_cmd_binder.set_handler("mms note", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_NOTE), tr("Send a one-line message to an authorized signer, identified by its label, or show any waiting unread notes")); m_cmd_binder.set_handler("mms show", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_SHOW), tr("Show detailed info about a single message")); m_cmd_binder.set_handler("mms set", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_SET), tr("Available options:\n " "auto-send <1|0>\n " @@ -3154,75 +3234,82 @@ simple_wallet::simple_wallet() tr(USAGE_MMS_SEND_SIGNER_CONFIG), tr("Send completed signer config to all other authorized signers")); m_cmd_binder.set_handler("mms start_auto_config", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_START_AUTO_CONFIG), tr("Start auto-config at the auto-config manager's wallet by issuing auto-config tokens and optionally set others' labels")); m_cmd_binder.set_handler("mms stop_auto_config", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_STOP_AUTO_CONFIG), tr("Delete any auto-config tokens and abort a auto-config process")); m_cmd_binder.set_handler("mms auto_config", - boost::bind(&simple_wallet::mms, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_AUTO_CONFIG), tr("Start auto-config by using the token received from the auto-config manager")); m_cmd_binder.set_handler("print_ring", - boost::bind(&simple_wallet::print_ring, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::print_ring, _1), tr(USAGE_PRINT_RING), tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)\n\n" "Output format:\n" "Key Image, \"absolute\", list of rings")); m_cmd_binder.set_handler("set_ring", - boost::bind(&simple_wallet::set_ring, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_ring, _1), tr(USAGE_SET_RING), tr("Set the ring used for a given key image, so it can be reused in a fork")); m_cmd_binder.set_handler("unset_ring", - boost::bind(&simple_wallet::unset_ring, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::unset_ring, _1), tr(USAGE_UNSET_RING), tr("Unsets the ring used for a given key image or transaction")); m_cmd_binder.set_handler("save_known_rings", - boost::bind(&simple_wallet::save_known_rings, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::save_known_rings, _1), tr(USAGE_SAVE_KNOWN_RINGS), tr("Save known rings to the shared rings database")); m_cmd_binder.set_handler("mark_output_spent", - boost::bind(&simple_wallet::blackball, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::blackball, _1), tr(USAGE_MARK_OUTPUT_SPENT), tr("Mark output(s) as spent so they never get selected as fake outputs in a ring")); m_cmd_binder.set_handler("mark_output_unspent", - boost::bind(&simple_wallet::unblackball, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::unblackball, _1), tr(USAGE_MARK_OUTPUT_UNSPENT), tr("Marks an output as unspent so it may get selected as a fake output in a ring")); m_cmd_binder.set_handler("is_output_spent", - boost::bind(&simple_wallet::blackballed, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::blackballed, _1), tr(USAGE_IS_OUTPUT_SPENT), tr("Checks whether an output is marked as spent")); m_cmd_binder.set_handler("freeze", - boost::bind(&simple_wallet::freeze, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::freeze, _1), tr(USAGE_FREEZE), tr("Freeze a single output by key image so it will not be used")); m_cmd_binder.set_handler("thaw", - boost::bind(&simple_wallet::thaw, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::thaw, _1), tr(USAGE_THAW), tr("Thaw a single output by key image so it may be used again")); m_cmd_binder.set_handler("frozen", - boost::bind(&simple_wallet::frozen, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::frozen, _1), tr(USAGE_FROZEN), tr("Checks whether a given output is currently frozen by key image")); + m_cmd_binder.set_handler("lock", + boost::bind(&simple_wallet::on_command, this, &simple_wallet::lock, _1), + tr(USAGE_LOCK), + tr("Lock the wallet console, requiring the wallet password to continue")); m_cmd_binder.set_handler("net_stats", - boost::bind(&simple_wallet::net_stats, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::net_stats, _1), tr(USAGE_NET_STATS), tr("Prints simple network stats")); m_cmd_binder.set_handler("welcome", - boost::bind(&simple_wallet::welcome, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::welcome, _1), tr(USAGE_WELCOME), tr("Prints basic info about Monero for first time users")); m_cmd_binder.set_handler("version", - boost::bind(&simple_wallet::version, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::version, _1), tr(USAGE_VERSION), tr("Returns version information")); m_cmd_binder.set_handler("help", - boost::bind(&simple_wallet::help, this, _1), + boost::bind(&simple_wallet::on_command, this, &simple_wallet::help, _1), tr(USAGE_HELP), tr("Show the help section or the documentation about a <command>.")); + m_cmd_binder.set_unknown_command_handler(boost::bind(&simple_wallet::on_command, this, &simple_wallet::on_unknown_command, _1)); + m_cmd_binder.set_empty_command_handler(boost::bind(&simple_wallet::on_empty_command, this)); + m_cmd_binder.set_cancel_handler(boost::bind(&simple_wallet::on_cancelled_command, this)); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::set_variable(const std::vector<std::string> &args) @@ -3278,6 +3365,12 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "track-uses = " << m_wallet->track_uses(); success_msg_writer() << "setup-background-mining = " << setup_background_mining_string; success_msg_writer() << "device-name = " << m_wallet->device_name(); + success_msg_writer() << "export-format = " << (m_wallet->export_format() == tools::wallet2::ExportFormat::Ascii ? "ascii" : "binary"); + success_msg_writer() << "inactivity-lock-timeout = " << m_wallet->inactivity_lock_timeout() +#ifdef _WIN32 + << " (disabled on Windows)" +#endif + ; return true; } else @@ -3334,8 +3427,10 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("segregation-height", set_segregation_height, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("ignore-fractional-outputs", set_ignore_fractional_outputs, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("track-uses", set_track_uses, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("inactivity-lock-timeout", set_inactivity_lock_timeout, tr("unsigned integer (seconds, 0 to disable)")); CHECK_SIMPLE_VARIABLE("setup-background-mining", set_setup_background_mining, tr("1/yes or 0/no")); CHECK_SIMPLE_VARIABLE("device-name", set_device_name, tr("<device_name[:device_spec]>")); + CHECK_SIMPLE_VARIABLE("export-format", set_export_format, tr("\"binary\" or \"ascii\"")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -4101,7 +4196,22 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } if (!m_wallet->is_trusted_daemon()) - message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); + { + message_writer(console_color_red, true) << (boost::format(tr("Warning: using an untrusted daemon at %s")) % m_wallet->get_daemon_address()).str(); + message_writer(console_color_red, true) << boost::format(tr("Using a third party daemon can be detrimental to your security and privacy")); + bool ssl = false; + if (m_wallet->check_connection(NULL, &ssl) && !ssl) + message_writer(console_color_red, true) << boost::format(tr("Using your own without SSL exposes your RPC traffic to monitoring")); + message_writer(console_color_red, true) << boost::format(tr("You are strongly encouraged to connect to the Monero network using your own daemon")); + message_writer(console_color_red, true) << boost::format(tr("If you or someone you trust are operating this daemon, you can use --trusted-daemon")); + + COMMAND_RPC_GET_INFO::request req; + COMMAND_RPC_GET_INFO::response res; + bool r = m_wallet->invoke_http_json("/get_info", req, res); + std::string err = interpret_rpc_response(r, res.status); + if (r && err.empty() && (res.was_bootstrap_ever_used || !res.bootstrap_daemon_address.empty())) + message_writer(console_color_red, true) << boost::format(tr("Moreover, a daemon is also less secure when running in bootstrap mode")); + } if (m_wallet->get_ring_database().empty()) fail_msg_writer() << tr("Failed to initialize ring database: privacy enhancing features will be inactive"); @@ -4123,6 +4233,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) tr("It is recommended that you do not use them, and ask recipients who ask for one to not endanger your privacy."); } + m_last_activity_time = time(NULL); return true; } //---------------------------------------------------------------------------------------------------- @@ -4961,12 +5072,16 @@ bool simple_wallet::save_bc(const std::vector<std::string>& args) //---------------------------------------------------------------------------------------------------- void simple_wallet::on_new_block(uint64_t height, const cryptonote::block& block) { + if (m_locked) + return; if (!m_auto_refresh_refreshing) m_refresh_progress_reporter.update(height, false); } //---------------------------------------------------------------------------------------------------- void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) { + if (m_locked) + return; message_writer(console_color_green, false) << "\r" << tr("Height ") << height << ", " << tr("txid ") << txid << ", " << @@ -4997,11 +5112,15 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, //---------------------------------------------------------------------------------------------------- void simple_wallet::on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) { + if (m_locked) + return; // Not implemented in CLI wallet } //---------------------------------------------------------------------------------------------------- void simple_wallet::on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) { + if (m_locked) + return; message_writer(console_color_magenta, false) << "\r" << tr("Height ") << height << ", " << tr("txid ") << txid << ", " << @@ -5015,10 +5134,14 @@ void simple_wallet::on_money_spent(uint64_t height, const crypto::hash &txid, co //---------------------------------------------------------------------------------------------------- void simple_wallet::on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) { + if (m_locked) + return; } //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char *reason) { + if (m_locked) + return boost::none; // can't ask for password from a background thread if (!m_in_manual_refresh.load(std::memory_order_relaxed)) { @@ -5027,9 +5150,7 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return boost::none; } -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); std::string msg = tr("Enter password"); if (reason && *reason) msg += std::string(" (") + reason + ")"; @@ -5050,9 +5171,7 @@ void simple_wallet::on_device_button_request(uint64_t code) //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request() { -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); std::string msg = tr("Enter device PIN"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN")); @@ -5066,9 +5185,7 @@ boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_reque return boost::none; } -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); std::string msg = tr("Enter device passphrase"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); @@ -5115,9 +5232,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo m_wallet->rescan_blockchain(reset == ResetHard, false, reset == ResetSoftKeepKI); } -#ifdef HAVE_READLINE - rdln::suspend_readline pause_readline; -#endif + PAUSE_READLINE(); message_writer() << tr("Starting refresh..."); @@ -5220,14 +5335,14 @@ bool simple_wallet::show_balance_unlocked(bool detailed) const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); uint64_t blocks_to_unlock; - uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, &blocks_to_unlock); + uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, false, &blocks_to_unlock); std::string unlock_time_message; if (blocks_to_unlock > 0) unlock_time_message = (boost::format(" (%lu block(s) to unlock)") % blocks_to_unlock).str(); - success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance(m_current_subaddress_account)) << ", " + success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance(m_current_subaddress_account, false)) << ", " << tr("unlocked balance: ") << print_money(unlocked_balance) << unlock_time_message << extra; - std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(m_current_subaddress_account); - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account); + std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(m_current_subaddress_account, false); + std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account, false); if (!detailed || balance_per_subaddress.empty()) return true; success_msg_writer() << tr("Balance per address:"); @@ -5675,6 +5790,67 @@ bool simple_wallet::prompt_if_old(const std::vector<tools::wallet2::pending_tx> } return true; } +void simple_wallet::check_for_inactivity_lock(bool user) +{ + if (m_locked) + { +#ifdef HAVE_READLINE + PAUSE_READLINE(); + rdln::clear_screen(); +#endif + tools::clear_screen(); + m_in_command = true; + if (!user) + { + const std::string speech = tr("I locked your Monero wallet to protect you while you were away"); + std::vector<std::pair<std::string, size_t>> lines = tools::split_string_by_width(speech, 45); + + size_t max_len = 0; + for (const auto &i: lines) + max_len = std::max(max_len, i.second); + const size_t n_u = max_len + 2; + tools::msg_writer() << " " << std::string(n_u, '_'); + for (size_t i = 0; i < lines.size(); ++i) + tools::msg_writer() << (i == 0 ? "/" : i == lines.size() - 1 ? "\\" : "|") << " " << lines[i].first << std::string(max_len - lines[i].second, ' ') << " " << (i == 0 ? "\\" : i == lines.size() - 1 ? "/" : "|"); + tools::msg_writer() << " " << std::string(n_u, '-') << std::endl << + " \\ (__)" << std::endl << + " \\ (oo)\\_______" << std::endl << + " (__)\\ )\\/\\" << std::endl << + " ||----w |" << std::endl << + " || ||" << std::endl << + "" << std::endl; + } + while (1) + { + tools::msg_writer() << tr("Locked due to inactivity. The wallet password is required to unlock the console."); + try + { + if (get_and_verify_password()) + break; + } + catch (...) { /* do nothing, just let the loop loop */ } + } + m_last_activity_time = time(NULL); + m_in_command = false; + m_locked = false; + } +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::on_command(bool (simple_wallet::*cmd)(const std::vector<std::string>&), const std::vector<std::string> &args) +{ + const time_t now = time(NULL); + time_t dt = now - m_last_activity_time; + m_last_activity_time = time(NULL); + + m_in_command = true; + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + m_last_activity_time = time(NULL); + m_in_command = false; + }); + + check_for_inactivity_lock(false); + return (this->*cmd)(args); +} //---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_, bool called_by_mms) { @@ -7312,7 +7488,7 @@ bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) { std::string sig_str = m_wallet->get_tx_proof(txid, info.address, info.is_subaddress, args.size() == 3 ? args[2] : ""); const std::string filename = "monero_tx_proof"; - if (epee::file_io_utils::save_string_to_file(filename, sig_str)) + if (m_wallet->save_to_file(filename, sig_str, true)) success_msg_writer() << tr("signature file saved to: ") << filename; else fail_msg_writer() << tr("failed to save signature file"); @@ -7440,7 +7616,7 @@ bool simple_wallet::check_tx_proof(const std::vector<std::string> &args) // read signature file std::string sig_str; - if (!epee::file_io_utils::load_file_to_string(args[2], sig_str)) + if (!m_wallet->load_from_file(args[2], sig_str)) { fail_msg_writer() << tr("failed to load signature file"); return true; @@ -7524,7 +7700,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) { const std::string sig_str = m_wallet->get_spend_proof(txid, args.size() == 2 ? args[1] : ""); const std::string filename = "monero_spend_proof"; - if (epee::file_io_utils::save_string_to_file(filename, sig_str)) + if (m_wallet->save_to_file(filename, sig_str, true)) success_msg_writer() << tr("signature file saved to: ") << filename; else fail_msg_writer() << tr("failed to save signature file"); @@ -7554,7 +7730,7 @@ bool simple_wallet::check_spend_proof(const std::vector<std::string> &args) return true; std::string sig_str; - if (!epee::file_io_utils::load_file_to_string(args[1], sig_str)) + if (!m_wallet->load_from_file(args[1], sig_str)) { fail_msg_writer() << tr("failed to load signature file"); return true; @@ -7613,7 +7789,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) { const std::string sig_str = m_wallet->get_reserve_proof(account_minreserve, args.size() == 2 ? args[1] : ""); const std::string filename = "monero_reserve_proof"; - if (epee::file_io_utils::save_string_to_file(filename, sig_str)) + if (m_wallet->save_to_file(filename, sig_str, true)) success_msg_writer() << tr("signature file saved to: ") << filename; else fail_msg_writer() << tr("failed to save signature file"); @@ -7648,7 +7824,7 @@ bool simple_wallet::check_reserve_proof(const std::vector<std::string> &args) } std::string sig_str; - if (!epee::file_io_utils::load_file_to_string(args[1], sig_str)) + if (!m_wallet->load_from_file(args[1], sig_str)) { fail_msg_writer() << tr("failed to load signature file"); return true; @@ -8313,6 +8489,35 @@ void simple_wallet::wallet_idle_thread() if (!m_idle_run.load(std::memory_order_relaxed)) break; +#ifndef _WIN32 + m_inactivity_checker.do_call(boost::bind(&simple_wallet::check_inactivity, this)); +#endif + m_refresh_checker.do_call(boost::bind(&simple_wallet::check_refresh, this)); + m_mms_checker.do_call(boost::bind(&simple_wallet::check_mms, this)); + + if (!m_idle_run.load(std::memory_order_relaxed)) + break; + m_idle_cond.wait_for(lock, boost::chrono::seconds(1)); + } +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::check_inactivity() +{ + // inactivity lock + if (!m_locked && !m_in_command) + { + const uint32_t seconds = m_wallet->inactivity_lock_timeout(); + if (seconds > 0 && time(NULL) - m_last_activity_time > seconds) + { + m_locked = true; + m_cmd_binder.cancel_input(); + } + } + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::check_refresh() +{ // auto refresh if (m_auto_refresh_enabled) { @@ -8327,7 +8532,11 @@ void simple_wallet::wallet_idle_thread() catch(...) {} m_auto_refresh_refreshing = false; } - + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::check_mms() +{ // Check for new MMS messages; // For simplicity auto message check is ALSO controlled by "m_auto_refresh_enabled" and has no // separate thread either; thread syncing is tricky enough with only this one idle thread here @@ -8335,15 +8544,13 @@ void simple_wallet::wallet_idle_thread() { check_for_messages(); } - - if (!m_idle_run.load(std::memory_order_relaxed)) - break; - m_idle_cond.wait_for(lock, boost::chrono::seconds(90)); - } + return true; } //---------------------------------------------------------------------------------------------------- std::string simple_wallet::get_prompt() const { + if (m_locked) + return std::string("[") + tr("locked due to inactivity") + "]"; std::string addr_start = m_wallet->get_subaddress_as_str({m_current_subaddress_account, 0}).substr(0, 6); std::string prompt = std::string("[") + tr("wallet") + " " + addr_start; if (!m_wallet->check_connection(NULL)) @@ -8536,7 +8743,7 @@ void simple_wallet::print_accounts() print_accounts(""); if (num_untagged_accounts < m_wallet->get_num_subaddress_accounts()) - success_msg_writer() << tr("\nGrand total:\n Balance: ") << print_money(m_wallet->balance_all()) << tr(", unlocked balance: ") << print_money(m_wallet->unlocked_balance_all()); + success_msg_writer() << tr("\nGrand total:\n Balance: ") << print_money(m_wallet->balance_all(false)) << tr(", unlocked balance: ") << print_money(m_wallet->unlocked_balance_all(false)); } //---------------------------------------------------------------------------------------------------- void simple_wallet::print_accounts(const std::string& tag) @@ -8566,11 +8773,11 @@ void simple_wallet::print_accounts(const std::string& tag) % (m_current_subaddress_account == account_index ? '*' : ' ') % account_index % m_wallet->get_subaddress_as_str({account_index, 0}).substr(0, 6) - % print_money(m_wallet->balance(account_index)) - % print_money(m_wallet->unlocked_balance(account_index)) + % print_money(m_wallet->balance(account_index, false)) + % print_money(m_wallet->unlocked_balance(account_index, false)) % m_wallet->get_subaddress_label({account_index, 0}); - total_balance += m_wallet->balance(account_index); - total_unlocked_balance += m_wallet->unlocked_balance(account_index); + total_balance += m_wallet->balance(account_index, false); + total_unlocked_balance += m_wallet->unlocked_balance(account_index, false); } success_msg_writer() << tr("----------------------------------------------------------------------------------"); success_msg_writer() << boost::format(tr("%15s %21s %21s")) % "Total" % print_money(total_balance) % print_money(total_unlocked_balance); @@ -9011,7 +9218,7 @@ bool simple_wallet::sign(const std::vector<std::string> &args) std::string filename = args[0]; std::string data; - bool r = epee::file_io_utils::load_file_to_string(filename, data); + bool r = m_wallet->load_from_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to read file ") << filename; @@ -9037,7 +9244,7 @@ bool simple_wallet::verify(const std::vector<std::string> &args) std::string signature= args[2]; std::string data; - bool r = epee::file_io_utils::load_file_to_string(filename, data); + bool r = m_wallet->load_from_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to read file ") << filename; @@ -9255,7 +9462,7 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args_) try { std::string data = m_wallet->export_outputs_to_str(all); - bool r = epee::file_io_utils::save_string_to_file(filename, data); + bool r = m_wallet->save_to_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -9288,7 +9495,7 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args) std::string filename = args[0]; std::string data; - bool r = epee::file_io_utils::load_file_to_string(filename, data); + bool r = m_wallet->load_from_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to read file ") << filename; @@ -9489,7 +9696,7 @@ void simple_wallet::commit_or_save(std::vector<tools::wallet2::pending_tx>& ptx_ tx_to_blob(ptx.tx, blob); const std::string blob_hex = epee::string_tools::buff_to_hex_nodelimer(blob); const std::string filename = "raw_monero_tx" + (ptx_vector.size() == 1 ? "" : ("_" + std::to_string(i++))); - if (epee::file_io_utils::save_string_to_file(filename, blob_hex)) + if (m_wallet->save_to_file(filename, blob_hex, true)) success_msg_writer(true) << tr("Transaction successfully saved to ") << filename << tr(", txid ") << txid; else fail_msg_writer() << tr("Failed to save transaction to ") << filename << tr(", txid ") << txid; @@ -9514,6 +9721,7 @@ int main(int argc, char* argv[]) std::locale::global(boost::locale::generator().generate("")); boost::filesystem::path::imbue(std::locale()); #endif + setlocale(LC_CTYPE, ""); po::options_description desc_params(wallet_args::tr("Wallet options")); tools::wallet2::init_options(desc_params); @@ -10274,7 +10482,7 @@ void simple_wallet::mms_export(const std::vector<std::string> &args) if (valid_id) { const std::string filename = "mms_message_content"; - if (epee::file_io_utils::save_string_to_file(filename, m.content)) + if (m_wallet->save_to_file(filename, m.content)) { success_msg_writer() << tr("Message content saved to: ") << filename; } diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 2de390666..22659e99e 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -44,6 +44,7 @@ #include "cryptonote_basic/cryptonote_basic_impl.h" #include "wallet/wallet2.h" #include "console_handler.h" +#include "math_helper.h" #include "wipeable_string.h" #include "common/i18n.h" #include "common/password.h" @@ -144,8 +145,10 @@ namespace cryptonote bool set_segregation_height(const std::vector<std::string> &args = std::vector<std::string>()); bool set_ignore_fractional_outputs(const std::vector<std::string> &args = std::vector<std::string>()); bool set_track_uses(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_inactivity_lock_timeout(const std::vector<std::string> &args = std::vector<std::string>()); bool set_setup_background_mining(const std::vector<std::string> &args = std::vector<std::string>()); bool set_device_name(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_export_format(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -244,9 +247,11 @@ namespace cryptonote bool freeze(const std::vector<std::string>& args); bool thaw(const std::vector<std::string>& args); bool frozen(const std::vector<std::string>& args); + bool lock(const std::vector<std::string>& args); bool net_stats(const std::vector<std::string>& args); bool welcome(const std::vector<std::string>& args); bool version(const std::vector<std::string>& args); + bool on_unknown_command(const std::vector<std::string>& args); bool cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func); uint64_t get_daemon_blockchain_height(std::string& err); @@ -263,6 +268,10 @@ namespace cryptonote std::pair<std::string, std::string> show_outputs_line(const std::vector<uint64_t> &heights, uint64_t blockchain_height, uint64_t highlight_height = std::numeric_limits<uint64_t>::max()) const; bool freeze_thaw(const std::vector<std::string>& args, bool freeze); bool prompt_if_old(const std::vector<tools::wallet2::pending_tx> &ptx_vector); + bool on_command(bool (simple_wallet::*cmd)(const std::vector<std::string>&), const std::vector<std::string> &args); + bool on_empty_command(); + bool on_cancelled_command(); + void check_for_inactivity_lock(bool user); struct transfer_view { @@ -310,6 +319,11 @@ namespace cryptonote void start_background_mining(); void stop_background_mining(); + // idle thread workers + bool check_inactivity(); + bool check_refresh(); + bool check_mms(); + //----------------- i_wallet2_callback --------------------- virtual void on_new_block(uint64_t height, const cryptonote::block& block); virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time); @@ -417,6 +431,14 @@ namespace cryptonote uint32_t m_current_subaddress_account; bool m_long_payment_id_support; + + std::atomic<time_t> m_last_activity_time; + std::atomic<bool> m_locked; + std::atomic<bool> m_in_command; + + epee::math_helper::once_a_time_seconds<1> m_inactivity_checker; + epee::math_helper::once_a_time_seconds<90> m_refresh_checker; + epee::math_helper::once_a_time_seconds<90> m_mms_checker; // MMS mms::message_store& get_message_store() const { return m_wallet->get_message_store(); }; diff --git a/src/version.cpp.in b/src/version.cpp.in index 28ce38df7..9121cdf85 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -2,6 +2,7 @@ #define DEF_MONERO_VERSION "0.14.1.2" #define DEF_MONERO_RELEASE_NAME "Boron Butterfly" #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG +#define DEF_MONERO_VERSION_IS_RELEASE @VERSION_IS_RELEASE@ #include "version.h" @@ -9,3 +10,4 @@ const char* const MONERO_VERSION_TAG = DEF_MONERO_VERSION_TAG; const char* const MONERO_VERSION = DEF_MONERO_VERSION; const char* const MONERO_RELEASE_NAME = DEF_MONERO_RELEASE_NAME; const char* const MONERO_VERSION_FULL = DEF_MONERO_VERSION_FULL; +const bool MONERO_VERSION_IS_RELEASE = DEF_MONERO_VERSION_IS_RELEASE; diff --git a/src/version.h b/src/version.h index d1d06c790..b740d3360 100644 --- a/src/version.h +++ b/src/version.h @@ -4,3 +4,4 @@ extern const char* const MONERO_VERSION_TAG; extern const char* const MONERO_VERSION; extern const char* const MONERO_RELEASE_NAME; extern const char* const MONERO_VERSION_FULL; +extern const bool MONERO_VERSION_IS_RELEASE; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 8da95de7b..b7efdd75c 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -424,7 +424,7 @@ bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> & { case BLACKBALL_BLACKBALL: MDEBUG("Marking output " << output.first << "/" << output.second << " as spent"); - dbr = mdb_cursor_put(cursor, &key, &data, MDB_APPENDDUP); + dbr = mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA); if (dbr == MDB_KEYEXIST) dbr = 0; break; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 9782e4b1e..476c68c74 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -40,6 +40,7 @@ #include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/preprocessor/stringize.hpp> +#include <openssl/evp.h> #include "include_base_utils.h" using namespace epee; @@ -135,9 +136,13 @@ using namespace cryptonote; #define DEFAULT_MIN_OUTPUT_COUNT 5 #define DEFAULT_MIN_OUTPUT_VALUE (2*COIN) +#define DEFAULT_INACTIVITY_LOCK_TIMEOUT 90 // a minute and a half + static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string MULTISIG_EXTRA_INFO_MAGIC = "MultisigxV1"; +static const std::string ASCII_OUTPUT_MAGIC = "MoneroAsciiDataV1"; + namespace { std::string get_default_ringdb_path() @@ -275,6 +280,7 @@ struct options { const command_line::arg_descriptor<std::string> tx_notify = { "tx-notify" , "Run a program for each new incoming transaction, '%s' will be replaced by the transaction hash" , "" }; const command_line::arg_descriptor<bool> no_dns = {"no-dns", tools::wallet2::tr("Do not use DNS"), false}; const command_line::arg_descriptor<bool> offline = {"offline", tools::wallet2::tr("Do not connect to a daemon, nor use DNS"), false}; + const command_line::arg_descriptor<std::string> extra_entropy = {"extra-entropy", tools::wallet2::tr("File containing extra entropy to initialize the PRNG (any data, aim for 256 bits of entropy to be useful, wihch typically means more than 256 bits of data)")}; }; void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file, std::string &mms_file) @@ -476,6 +482,15 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (command_line::get_arg(vm, opts.offline)) wallet->set_offline(); + const std::string extra_entropy = command_line::get_arg(vm, opts.extra_entropy); + if (!extra_entropy.empty()) + { + std::string data; + THROW_WALLET_EXCEPTION_IF(!epee::file_io_utils::load_file_to_string(extra_entropy, data), + tools::error::wallet_internal_error, "Failed to load extra entropy from " + extra_entropy); + add_extra_entropy_thread_safe(data.data(), data.size()); + } + try { if (!command_line::is_arg_defaulted(vm, opts.tx_notify)) @@ -1114,6 +1129,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_segregation_height(0), m_ignore_fractional_outputs(true), m_track_uses(false), + m_inactivity_lock_timeout(DEFAULT_INACTIVITY_LOCK_TIMEOUT), m_setup_background_mining(BackgroundMiningMaybe), m_is_initialized(false), m_kdf_rounds(kdf_rounds), @@ -1143,7 +1159,8 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_device_last_key_image_sync(0), m_use_dns(true), m_offline(false), - m_rpc_version(0) + m_rpc_version(0), + m_export_format(ExportFormat::Binary) { } @@ -1200,6 +1217,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.tx_notify); command_line::add_arg(desc_params, opts.no_dns); command_line::add_arg(desc_params, opts.offline); + command_line::add_arg(desc_params, opts.extra_entropy); } std::pair<std::unique_ptr<wallet2>, tools::password_container> wallet2::make_from_json(const boost::program_options::variables_map& vm, bool unattended, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) @@ -1507,7 +1525,9 @@ void wallet2::set_subaddress_label(const cryptonote::subaddress_index& index, co //---------------------------------------------------------------------------------------------------- void wallet2::set_subaddress_lookahead(size_t major, size_t minor) { + THROW_WALLET_EXCEPTION_IF(major == 0, error::wallet_internal_error, "Subaddress major lookahead may not be zero"); THROW_WALLET_EXCEPTION_IF(major > 0xffffffff, error::wallet_internal_error, "Subaddress major lookahead is too large"); + THROW_WALLET_EXCEPTION_IF(minor == 0, error::wallet_internal_error, "Subaddress minor lookahead may not be zero"); THROW_WALLET_EXCEPTION_IF(minor > 0xffffffff, error::wallet_internal_error, "Subaddress minor lookahead is too large"); m_subaddress_lookahead_major = major; m_subaddress_lookahead_minor = minor; @@ -1523,6 +1543,7 @@ bool wallet2::is_deprecated() const //---------------------------------------------------------------------------------------------------- void wallet2::set_spent(size_t idx, uint64_t height) { + CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid index"); transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Setting SPENT at " << height << ": ki " << td.m_key_image << ", amount " << print_money(td.m_amount)); td.m_spent = true; @@ -1531,12 +1552,32 @@ void wallet2::set_spent(size_t idx, uint64_t height) //---------------------------------------------------------------------------------------------------- void wallet2::set_unspent(size_t idx) { + CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid index"); transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Setting UNSPENT: ki " << td.m_key_image << ", amount " << print_money(td.m_amount)); td.m_spent = false; td.m_spent_height = 0; } //---------------------------------------------------------------------------------------------------- +bool wallet2::is_spent(const transfer_details &td, bool strict) const +{ + if (strict) + { + return td.m_spent && td.m_spent_height > 0; + } + else + { + return td.m_spent; + } +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::is_spent(size_t idx, bool strict) const +{ + CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid index"); + const transfer_details &td = m_transfers[idx]; + return is_spent(td, strict); +} +//---------------------------------------------------------------------------------------------------- void wallet2::freeze(size_t idx) { CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid transfer_details index"); @@ -3274,7 +3315,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo m_first_refresh_done = true; - LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); + LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all(false)) << ", unlocked: " << print_money(unlocked_balance_all(false))); } //---------------------------------------------------------------------------------------------------- bool wallet2::refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok) @@ -3635,6 +3676,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetInt(m_track_uses ? 1 : 0); json.AddMember("track_uses", value2, json.GetAllocator()); + value2.SetInt(m_inactivity_lock_timeout); + json.AddMember("inactivity_lock_timeout", value2, json.GetAllocator()); + value2.SetInt(m_setup_background_mining); json.AddMember("setup_background_mining", value2, json.GetAllocator()); @@ -3647,6 +3691,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetInt(m_original_keys_available ? 1 : 0); json.AddMember("original_keys_available", value2, json.GetAllocator()); + value2.SetInt(m_export_format); + json.AddMember("export_format", value2, json.GetAllocator()); + value2.SetUint(1); json.AddMember("encrypted_secret_keys", value2, json.GetAllocator()); @@ -3684,7 +3731,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable std::string tmp_file_name = keys_file_name + ".new"; std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); - r = r && epee::file_io_utils::save_string_to_file(tmp_file_name, buf); + r = r && save_to_file(tmp_file_name, buf); CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); unlock_keys_file(); @@ -3741,7 +3788,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ wallet2::keys_file_data keys_file_data; std::string buf; bool encrypted_secret_keys = false; - bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); + bool r = load_from_file(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); // Decrypt the contents @@ -3754,7 +3801,6 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); if (json.Parse(account_data.c_str()).HasParseError() || !json.IsObject()) crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); - // The contents should be JSON if the wallet follows the new format. if (json.Parse(account_data.c_str()).HasParseError()) { @@ -3789,10 +3835,12 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_segregation_height = 0; m_ignore_fractional_outputs = true; m_track_uses = false; + m_inactivity_lock_timeout = DEFAULT_INACTIVITY_LOCK_TIMEOUT; m_setup_background_mining = BackgroundMiningMaybe; m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; m_original_keys_available = false; + m_export_format = ExportFormat::Binary; m_device_name = ""; m_device_derivation_path = ""; m_key_device_type = hw::device::device_type::SOFTWARE; @@ -3944,6 +3992,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_ignore_fractional_outputs = field_ignore_fractional_outputs; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, track_uses, int, Int, false, false); m_track_uses = field_track_uses; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, inactivity_lock_timeout, uint32_t, Uint, false, DEFAULT_INACTIVITY_LOCK_TIMEOUT); + m_inactivity_lock_timeout = field_inactivity_lock_timeout; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, setup_background_mining, BackgroundMiningSetupType, Int, false, BackgroundMiningMaybe); m_setup_background_mining = field_setup_background_mining; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_major, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MAJOR); @@ -3954,6 +4004,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); encrypted_secret_keys = field_encrypted_secret_keys; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, export_format, ExportFormat, Int, false, Binary); + m_export_format = field_export_format; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_name, std::string, String, false, std::string()); if (m_device_name.empty()) { @@ -4101,7 +4154,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip wallet2::keys_file_data keys_file_data; std::string buf; bool encrypted_secret_keys = false; - bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); + bool r = load_from_file(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); // Decrypt the contents @@ -4186,7 +4239,7 @@ void wallet2::create_keys_file(const std::string &wallet_, bool watch_only, cons if (create_address_file) { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); + r = save_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype), true); if(!r) MERROR("String with address text not saved"); } } @@ -4208,7 +4261,7 @@ bool wallet2::query_device(hw::device::device_type& device_type, const std::stri rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; - bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); + bool r = load_from_file(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); // Decrypt the contents @@ -4757,7 +4810,7 @@ std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &passwor if (boost::filesystem::exists(m_wallet_file + ".address.txt")) { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); + r = save_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype), true); if(!r) MERROR("String with address text not saved"); } } @@ -5266,7 +5319,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass { wallet2::cache_file_data cache_file_data; std::string buf; - bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf, std::numeric_limits<size_t>::max()); + bool r = load_from_file(m_wallet_file, buf, std::numeric_limits<size_t>::max()); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); // try to read it as an encrypted cache @@ -5501,7 +5554,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas { // save address to the new file const std::string address_file = m_wallet_file + ".address.txt"; - r = file_io_utils::save_string_to_file(address_file, m_account.get_public_address_str(m_nettype)); + r = save_to_file(address_file, m_account.get_public_address_str(m_nettype), true); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_wallet_file); } // remove old wallet file @@ -5536,7 +5589,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas binary_archive<true> oar(oss); bool success = ::serialization::serialize(oar, cache_file_data); if (success) { - success = epee::file_io_utils::save_string_to_file(new_file, oss.str()); + success = save_to_file(new_file, oss.str()); } THROW_WALLET_EXCEPTION_IF(!success, error::file_save_error, new_file); #else @@ -5562,24 +5615,24 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::balance(uint32_t index_major) const +uint64_t wallet2::balance(uint32_t index_major, bool strict) const { uint64_t amount = 0; if(m_light_wallet) return m_light_wallet_unlocked_balance; - for (const auto& i : balance_per_subaddress(index_major)) + for (const auto& i : balance_per_subaddress(index_major, strict)) amount += i.second; return amount; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::unlocked_balance(uint32_t index_major, uint64_t *blocks_to_unlock) const +uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t *blocks_to_unlock) const { uint64_t amount = 0; if (blocks_to_unlock) *blocks_to_unlock = 0; if(m_light_wallet) return m_light_wallet_balance; - for (const auto& i : unlocked_balance_per_subaddress(index_major)) + for (const auto& i : unlocked_balance_per_subaddress(index_major, strict)) { amount += i.second.first; if (blocks_to_unlock && i.second.second > *blocks_to_unlock) @@ -5588,12 +5641,12 @@ uint64_t wallet2::unlocked_balance(uint32_t index_major, uint64_t *blocks_to_unl return amount; } //---------------------------------------------------------------------------------------------------- -std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_major) const +std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_major, bool strict) const { std::map<uint32_t, uint64_t> amount_per_subaddr; for (const auto& td: m_transfers) { - if (td.m_subaddr_index.major == index_major && !td.m_spent && !td.m_frozen) + if (td.m_subaddr_index.major == index_major && !is_spent(td, strict) && !td.m_frozen) { auto found = amount_per_subaddr.find(td.m_subaddr_index.minor); if (found == amount_per_subaddr.end()) @@ -5602,8 +5655,10 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo found->second += td.amount(); } } - for (const auto& utx: m_unconfirmed_txs) + if (!strict) { + for (const auto& utx: m_unconfirmed_txs) + { if (utx.second.m_subaddr_account == index_major && utx.second.m_state != wallet2::unconfirmed_transfer_details::failed) { // all changes go to 0-th subaddress (in the current subaddress account) @@ -5613,17 +5668,18 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo else found->second += utx.second.m_change; } + } } return amount_per_subaddr; } //---------------------------------------------------------------------------------------------------- -std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major) const +std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major, bool strict) const { std::map<uint32_t, std::pair<uint64_t, uint64_t>> amount_per_subaddr; const uint64_t blockchain_height = get_blockchain_current_height(); for(const transfer_details& td: m_transfers) { - if(td.m_subaddr_index.major == index_major && !td.m_spent && !td.m_frozen) + if(td.m_subaddr_index.major == index_major && !is_spent(td, strict) && !td.m_frozen) { uint64_t amount = 0, blocks_to_unlock = 0; if (is_transfer_unlocked(td)) @@ -5652,15 +5708,15 @@ std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_ return amount_per_subaddr; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::balance_all() const +uint64_t wallet2::balance_all(bool strict) const { uint64_t r = 0; for (uint32_t index_major = 0; index_major < get_num_subaddress_accounts(); ++index_major) - r += balance(index_major); + r += balance(index_major, strict); return r; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::unlocked_balance_all(uint64_t *blocks_to_unlock) const +uint64_t wallet2::unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock) const { uint64_t r = 0; if (blocks_to_unlock) @@ -5668,7 +5724,7 @@ uint64_t wallet2::unlocked_balance_all(uint64_t *blocks_to_unlock) const for (uint32_t index_major = 0; index_major < get_num_subaddress_accounts(); ++index_major) { uint64_t local_blocks_to_unlock; - r += unlocked_balance(index_major, blocks_to_unlock ? &local_blocks_to_unlock : NULL); + r += unlocked_balance(index_major, strict, blocks_to_unlock ? &local_blocks_to_unlock : NULL); if (blocks_to_unlock) *blocks_to_unlock = std::max(*blocks_to_unlock, local_blocks_to_unlock); } @@ -6145,8 +6201,8 @@ void wallet2::commit_tx(pending_tx& ptx) //fee includes dust if dust policy specified it. LOG_PRINT_L1("Transaction successfully sent. <" << txid << ">" << ENDL << "Commission: " << print_money(ptx.fee) << " (dust sent to dust addr: " << print_money((ptx.dust_added_to_fee ? 0 : ptx.dust)) << ")" << ENDL - << "Balance: " << print_money(balance(ptx.construction_data.subaddr_account)) << ENDL - << "Unlocked: " << print_money(unlocked_balance(ptx.construction_data.subaddr_account)) << ENDL + << "Balance: " << print_money(balance(ptx.construction_data.subaddr_account, false)) << ENDL + << "Unlocked: " << print_money(unlocked_balance(ptx.construction_data.subaddr_account, false)) << ENDL << "Please, wait for confirmation for your balance to be unlocked."); } @@ -6164,7 +6220,7 @@ bool wallet2::save_tx(const std::vector<pending_tx>& ptx_vector, const std::stri std::string ciphertext = dump_tx_to_str(ptx_vector); if (ciphertext.empty()) return false; - return epee::file_io_utils::save_string_to_file(filename, ciphertext); + return save_to_file(filename, ciphertext); } //---------------------------------------------------------------------------------------------------- std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) const @@ -6206,7 +6262,7 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx LOG_PRINT_L0("File " << unsigned_filename << " does not exist: " << errcode); return false; } - if (!epee::file_io_utils::load_file_to_string(unsigned_filename.c_str(), s)) + if (!load_from_file(unsigned_filename.c_str(), s)) { LOG_PRINT_L0("Failed to load from " << unsigned_filename); return false; @@ -6314,7 +6370,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin // normally, the tx keys are saved in commit_tx, when the tx is actually sent to the daemon. // we can't do that here since the tx will be sent from the compromised wallet, which we don't want // to see that info, so we save it here - if (store_tx_info() && ptx.tx_key != crypto::null_skey) + if (store_tx_info() && tx_key != crypto::null_skey) { const crypto::hash txid = get_transaction_hash(ptx.tx); m_tx_keys.insert(std::make_pair(txid, tx_key)); @@ -6427,7 +6483,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f return false; } - if (!epee::file_io_utils::save_string_to_file(signed_filename, ciphertext)) + if (!save_to_file(signed_filename, ciphertext)) { LOG_PRINT_L0("Failed to save file to " << signed_filename); return false; @@ -6439,7 +6495,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f { std::string tx_as_hex = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(signed_txes.ptx[i].tx)); std::string raw_filename = signed_filename + "_raw" + (signed_txes.ptx.size() == 1 ? "" : ("_" + std::to_string(i))); - if (!epee::file_io_utils::save_string_to_file(raw_filename, tx_as_hex)) + if (!save_to_file(raw_filename, tx_as_hex)) { LOG_PRINT_L0("Failed to save file to " << raw_filename); return false; @@ -6487,7 +6543,7 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal return false; } - if (!epee::file_io_utils::load_file_to_string(signed_filename.c_str(), s)) + if (!load_from_file(signed_filename.c_str(), s)) { LOG_PRINT_L0("Failed to load from " << signed_filename); return false; @@ -6618,7 +6674,7 @@ bool wallet2::save_multisig_tx(const multisig_tx_set &txs, const std::string &fi std::string ciphertext = save_multisig_tx(txs); if (ciphertext.empty()) return false; - return epee::file_io_utils::save_string_to_file(filename, ciphertext); + return save_to_file(filename, ciphertext); } //---------------------------------------------------------------------------------------------------- wallet2::multisig_tx_set wallet2::make_multisig_tx_set(const std::vector<pending_tx>& ptx_vector) const @@ -6646,7 +6702,7 @@ bool wallet2::save_multisig_tx(const std::vector<pending_tx>& ptx_vector, const std::string ciphertext = save_multisig_tx(ptx_vector); if (ciphertext.empty()) return false; - return epee::file_io_utils::save_string_to_file(filename, ciphertext); + return save_to_file(filename, ciphertext); } //---------------------------------------------------------------------------------------------------- bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx_set &exported_txs) const @@ -6737,7 +6793,7 @@ bool wallet2::load_multisig_tx_from_file(const std::string &filename, multisig_t LOG_PRINT_L0("File " << filename << " does not exist: " << errcode); return false; } - if (!epee::file_io_utils::load_file_to_string(filename.c_str(), s)) + if (!load_from_file(filename.c_str(), s)) { LOG_PRINT_L0("Failed to load from " << filename); return false; @@ -7226,9 +7282,7 @@ bool wallet2::unset_ring(const crypto::hash &txid) bool ok = invoke_http_json("/gettransactions", req, res, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to get transaction from daemon"); - if (res.txs.empty()) - return false; - THROW_WALLET_EXCEPTION_IF(res.txs.size(), error::wallet_internal_error, "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "Failed to get transaction from daemon"); cryptonote::transaction tx; crypto::hash tx_hash; @@ -8529,7 +8583,7 @@ std::vector<size_t> wallet2::pick_preferred_rct_inputs(uint64_t needed_money, ui for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && td.is_rct() && td.amount() >= needed_money && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) + if (!is_spent(td, false) && !td.m_frozen && td.is_rct() && td.amount() >= needed_money && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { LOG_PRINT_L2("We can use " << i << " alone: " << print_money(td.amount())); picks.push_back(i); @@ -8544,13 +8598,13 @@ std::vector<size_t> wallet2::pick_preferred_rct_inputs(uint64_t needed_money, ui for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && !td.m_key_image_partial && td.is_rct() && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) + if (!is_spent(td, false) && !td.m_frozen && !td.m_key_image_partial && td.is_rct() && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { LOG_PRINT_L2("Considering input " << i << ", " << print_money(td.amount())); for (size_t j = i + 1; j < m_transfers.size(); ++j) { const transfer_details& td2 = m_transfers[j]; - if (!td2.m_spent && !td2.m_frozen && !td.m_key_image_partial && td2.is_rct() && td.amount() + td2.amount() >= needed_money && is_transfer_unlocked(td2) && td2.m_subaddr_index == td.m_subaddr_index) + if (!is_spent(td2, false) && !td2.m_frozen && !td.m_key_image_partial && td2.is_rct() && td.amount() + td2.amount() >= needed_money && is_transfer_unlocked(td2) && td2.m_subaddr_index == td.m_subaddr_index) { // update our picks if those outputs are less related than any we // already found. If the same, don't update, and oldest suitable outputs @@ -9205,8 +9259,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // throw if attempting a transaction with no money THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination); - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account); - std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account); + std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); + std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false); if (subaddr_indices.empty()) // "index=<N1>[,<N2>,...]" wasn't specified -> use all the indices with non-zero unlocked balance { @@ -9252,7 +9306,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp MDEBUG("Ignoring output " << i << " of amount " << print_money(td.amount()) << " which is below threshold " << print_money(fractional_threshold)); continue; } - if (!td.m_spent && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) + if (!is_spent(td, false) && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { const uint32_t index_minor = td.m_subaddr_index.minor; auto find_predicate = [&index_minor](const std::pair<uint32_t, std::vector<size_t>>& x) { return x.first == index_minor; }; @@ -9379,7 +9433,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // if we need to spend money and don't have any left, we fail if (unused_dust_indices->empty() && unused_transfers_indices->empty()) { LOG_PRINT_L2("No more outputs to choose from"); - THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account), needed_money, accumulated_fee + needed_fee); + THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account, false), needed_money, accumulated_fee + needed_fee); } // get a random unspent output and use it to pay part (or all) of the current destination (and maybe next one, etc) @@ -9597,7 +9651,7 @@ skip_tx: if (adding_fee) { LOG_PRINT_L1("We ran out of outputs while trying to gather final fee"); - THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account), needed_money, accumulated_fee + needed_fee); + THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account, false), needed_money, accumulated_fee + needed_fee); } LOG_PRINT_L1("Done creating " << txes.size() << " transactions, " << print_money(accumulated_fee) << @@ -9732,7 +9786,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below std::vector<size_t> unused_dust_indices; const bool use_rct = use_fork_rules(4, 0); - THROW_WALLET_EXCEPTION_IF(unlocked_balance(subaddr_account) == 0, error::wallet_internal_error, "No unlocked balance in the entire wallet"); + THROW_WALLET_EXCEPTION_IF(unlocked_balance(subaddr_account, false) == 0, error::wallet_internal_error, "No unlocked balance in the entire wallet"); std::map<uint32_t, std::pair<std::vector<size_t>, std::vector<size_t>>> unused_transfer_dust_indices_per_subaddr; @@ -9741,7 +9795,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && (subaddr_indices.empty() || subaddr_indices.count(td.m_subaddr_index.minor) == 1)) + if (!is_spent(td, false) && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && (subaddr_indices.empty() || subaddr_indices.count(td.m_subaddr_index.minor) == 1)) { fund_found = true; if (below == 0 || td.amount() < below) @@ -9789,7 +9843,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (td.m_key_image_known && td.m_key_image == ki && !td.m_spent && !td.m_frozen && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td)) + if (td.m_key_image_known && td.m_key_image == ki && !is_spent(td, false) && !td.m_frozen && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td)) { if (td.is_rct() || is_valid_decomposed_amount(td.amount())) unused_transfers_indices.push_back(i); @@ -10155,7 +10209,7 @@ std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(c size_t n = 0; for (transfer_container::const_iterator i = m_transfers.begin(); i != m_transfers.end(); ++i, ++n) { - if (i->m_spent) + if (is_spent(*i, false)) continue; if (i->m_frozen) continue; @@ -10169,12 +10223,12 @@ std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(c return outputs; } //---------------------------------------------------------------------------------------------------- -std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const +std::vector<uint64_t> wallet2::get_unspent_amounts_vector(bool strict) const { std::set<uint64_t> set; for (const auto &td: m_transfers) { - if (!td.m_spent && !td.m_frozen) + if (!is_spent(td, strict) && !td.m_frozen) set.insert(td.is_rct() ? 0 : td.amount()); } std::vector<uint64_t> vector; @@ -10192,7 +10246,7 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); if (is_trusted_daemon()) - req_t.amounts = get_unspent_amounts_vector(); + req_t.amounts = get_unspent_amounts_vector(false); req_t.min_count = count; req_t.max_count = 0; req_t.unlocked = unlocked; @@ -11090,8 +11144,8 @@ bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, uint64_t>> &account_minreserve, const std::string &message) { THROW_WALLET_EXCEPTION_IF(m_watch_only || m_multisig, error::wallet_internal_error, "Reserve proof can only be generated by a full wallet"); - THROW_WALLET_EXCEPTION_IF(balance_all() == 0, error::wallet_internal_error, "Zero balance"); - THROW_WALLET_EXCEPTION_IF(account_minreserve && balance(account_minreserve->first) < account_minreserve->second, error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(balance_all(true) == 0, error::wallet_internal_error, "Zero balance"); + THROW_WALLET_EXCEPTION_IF(account_minreserve && balance(account_minreserve->first, true) < account_minreserve->second, error::wallet_internal_error, "Not enough balance in this account for the requested minimum reserve amount"); // determine which outputs to include in the proof @@ -11099,7 +11153,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details &td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && (!account_minreserve || account_minreserve->first == td.m_subaddr_index.major)) + if (!is_spent(td, true) && !td.m_frozen && (!account_minreserve || account_minreserve->first == td.m_subaddr_index.major)) selected_transfers.push_back(i); } @@ -11633,7 +11687,7 @@ bool wallet2::export_key_images(const std::string &filename, bool all) const // encrypt data, keep magic plaintext PERF_TIMER(export_key_images_encrypt); std::string ciphertext = encrypt_with_view_secret_key(data); - return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); + return save_to_file(filename, magic + ciphertext); } //---------------------------------------------------------------------------------------------------- @@ -11698,7 +11752,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent { PERF_TIMER(import_key_images_fsu); std::string data; - bool r = epee::file_io_utils::load_file_to_string(filename, data); + bool r = load_from_file(filename, data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, std::string(tr("failed to read file ")) + filename); @@ -13102,6 +13156,83 @@ void wallet2::throw_on_rpc_response_error(const boost::optional<std::string> &st THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, m_trusted_daemon ? *status : "daemon error"); } //---------------------------------------------------------------------------------------------------- + +bool wallet2::save_to_file(const std::string& path_to_file, const std::string& raw, bool is_printable) const +{ + if (is_printable || m_export_format == ExportFormat::Binary) + { + return epee::file_io_utils::save_string_to_file(path_to_file, raw); + } + + FILE *fp = fopen(path_to_file.c_str(), "w+"); + // Save the result b/c we need to close the fp before returning success/failure. + int write_result = PEM_write(fp, ASCII_OUTPUT_MAGIC.c_str(), "", (const unsigned char *) raw.c_str(), raw.length()); + fclose(fp); + + if (write_result == 0) + { + return false; + } + else + { + return true; + } +} +//---------------------------------------------------------------------------------------------------- + +bool wallet2::load_from_file(const std::string& path_to_file, std::string& target_str, + size_t max_size) +{ + std::string data; + bool r = epee::file_io_utils::load_file_to_string(path_to_file, data, max_size); + if (!r) + { + return false; + } + + if (!boost::algorithm::contains(boost::make_iterator_range(data.begin(), data.end()), ASCII_OUTPUT_MAGIC)) + { + // It's NOT our ascii dump. + target_str = std::move(data); + return true; + } + + // Creating a BIO and calling PEM_read_bio instead of simpler PEM_read + // to avoid reading the file from disk twice. + BIO* b = BIO_new_mem_buf((const void*) data.data(), data.length()); + + char *name = NULL; + char *header = NULL; + unsigned char *openssl_data = NULL; + long len = 0; + + // Save the result b/c we need to free the data before returning success/failure. + int success = PEM_read_bio(b, &name, &header, &openssl_data, &len); + + try + { + target_str = std::string((const char*) openssl_data, len); + } + catch (...) + { + success = 0; + } + + OPENSSL_free((void *) name); + OPENSSL_free((void *) header); + OPENSSL_free((void *) openssl_data); + BIO_free(b); + + if (success == 0) + { + return false; + } + else + { + return true; + } +} +//---------------------------------------------------------------------------------------------------- void wallet2::hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const { KECCAK_CTX state; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index c2e34dd76..95f6f507a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -227,6 +227,11 @@ private: BackgroundMiningNo = 2, }; + enum ExportFormat { + Binary = 0, + Ascii, + }; + static const char* tr(const char* str); static bool has_testnet_option(const boost::program_options::variables_map& vm); @@ -799,14 +804,14 @@ private: bool reconnect_device(); // locked & unlocked balance of given or current subaddress account - uint64_t balance(uint32_t subaddr_index_major) const; - uint64_t unlocked_balance(uint32_t subaddr_index_major, uint64_t *blocks_to_unlock = NULL) const; + uint64_t balance(uint32_t subaddr_index_major, bool strict) const; + uint64_t unlocked_balance(uint32_t subaddr_index_major, bool strict, uint64_t *blocks_to_unlock = NULL) const; // locked & unlocked balance per subaddress of given or current subaddress account - std::map<uint32_t, uint64_t> balance_per_subaddress(uint32_t subaddr_index_major) const; - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major) const; + std::map<uint32_t, uint64_t> balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; + std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; // all locked & unlocked balances of all subaddress accounts - uint64_t balance_all() const; - uint64_t unlocked_balance_all(uint64_t *blocks_to_unlock = NULL) const; + uint64_t balance_all(bool strict) const; + uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL) const; template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, @@ -1046,10 +1051,14 @@ private: void track_uses(bool value) { m_track_uses = value; } BackgroundMiningSetupType setup_background_mining() const { return m_setup_background_mining; } void setup_background_mining(BackgroundMiningSetupType value) { m_setup_background_mining = value; } + uint32_t inactivity_lock_timeout() const { return m_inactivity_lock_timeout; } + void inactivity_lock_timeout(uint32_t seconds) { m_inactivity_lock_timeout = seconds; } const std::string & device_name() const { return m_device_name; } void device_name(const std::string & device_name) { m_device_name = device_name; } const std::string & device_derivation_path() const { return m_device_derivation_path; } void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } + const ExportFormat & export_format() const { return m_export_format; } + inline void set_export_format(const ExportFormat& export_format) { m_export_format = export_format; } bool get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); @@ -1300,6 +1309,9 @@ private: bool frozen(const crypto::key_image &ki) const; bool frozen(const transfer_details &td) const; + bool save_to_file(const std::string& path_to_file, const std::string& binary, bool is_printable = false) const; + static bool load_from_file(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000); + uint64_t get_bytes_sent() const; uint64_t get_bytes_received() const; @@ -1365,12 +1377,14 @@ private: void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; uint64_t get_upper_transaction_weight_limit() const; - std::vector<uint64_t> get_unspent_amounts_vector() const; + std::vector<uint64_t> get_unspent_amounts_vector(bool strict) const; uint64_t get_dynamic_base_fee_estimate() const; float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const; std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices) const; void set_spent(size_t idx, uint64_t height); void set_unspent(size_t idx); + bool is_spent(const transfer_details &td, bool strict = true) const; + bool is_spent(size_t idx, bool strict = true) const; void get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; @@ -1496,6 +1510,7 @@ private: uint64_t m_segregation_height; bool m_ignore_fractional_outputs; bool m_track_uses; + uint32_t m_inactivity_lock_timeout; BackgroundMiningSetupType m_setup_background_mining; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; @@ -1546,6 +1561,8 @@ private: std::shared_ptr<tools::Notify> m_tx_notify; std::unique_ptr<wallet_device_callback> m_device_callback; + + ExportFormat m_export_format; }; } BOOST_CLASS_VERSION(tools::wallet2, 28) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index c64b662f3..896f8f48e 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -36,6 +36,7 @@ #include "include_base_utils.h" using namespace epee; +#include "version.h" #include "wallet_rpc_server.h" #include "wallet/wallet_args.h" #include "common/command_line.h" @@ -419,8 +420,8 @@ namespace tools if (!m_wallet) return not_open(er); try { - res.balance = req.all_accounts ? m_wallet->balance_all() : m_wallet->balance(req.account_index); - res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(&res.blocks_to_unlock) : m_wallet->unlocked_balance(req.account_index, &res.blocks_to_unlock); + res.balance = req.all_accounts ? m_wallet->balance_all(req.strict) : m_wallet->balance(req.account_index, req.strict); + res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(req.strict, &res.blocks_to_unlock) : m_wallet->unlocked_balance(req.account_index, req.strict, &res.blocks_to_unlock); res.multisig_import_needed = m_wallet->multisig() && m_wallet->has_multisig_partial_key_images(); std::map<uint32_t, std::map<uint32_t, uint64_t>> balance_per_subaddress_per_account; std::map<uint32_t, std::map<uint32_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress_per_account; @@ -428,14 +429,14 @@ namespace tools { for (uint32_t account_index = 0; account_index < m_wallet->get_num_subaddress_accounts(); ++account_index) { - balance_per_subaddress_per_account[account_index] = m_wallet->balance_per_subaddress(account_index); - unlocked_balance_per_subaddress_per_account[account_index] = m_wallet->unlocked_balance_per_subaddress(account_index); + balance_per_subaddress_per_account[account_index] = m_wallet->balance_per_subaddress(account_index, req.strict); + unlocked_balance_per_subaddress_per_account[account_index] = m_wallet->unlocked_balance_per_subaddress(account_index, req.strict); } } else { - balance_per_subaddress_per_account[req.account_index] = m_wallet->balance_per_subaddress(req.account_index); - unlocked_balance_per_subaddress_per_account[req.account_index] = m_wallet->unlocked_balance_per_subaddress(req.account_index); + balance_per_subaddress_per_account[req.account_index] = m_wallet->balance_per_subaddress(req.account_index, req.strict); + unlocked_balance_per_subaddress_per_account[req.account_index] = m_wallet->unlocked_balance_per_subaddress(req.account_index, req.strict); } std::vector<tools::wallet2::transfer_details> transfers; m_wallet->get_transfers(transfers); @@ -593,8 +594,8 @@ namespace tools wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::subaddress_account_info info; info.account_index = subaddr_index.major; info.base_address = m_wallet->get_subaddress_as_str(subaddr_index); - info.balance = m_wallet->balance(subaddr_index.major); - info.unlocked_balance = m_wallet->unlocked_balance(subaddr_index.major); + info.balance = m_wallet->balance(subaddr_index.major, req.strict_balances); + info.unlocked_balance = m_wallet->unlocked_balance(subaddr_index.major, req.strict_balances); info.label = m_wallet->get_subaddress_label(subaddr_index); info.tag = account_tags.second[subaddr_index.major]; res.subaddress_accounts.push_back(info); @@ -4188,6 +4189,7 @@ namespace tools bool wallet_rpc_server::on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx) { res.version = WALLET_RPC_VERSION; + res.release = MONERO_VERSION_IS_RELEASE; return true; } //------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 4504ac752..e9b8b60f5 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 13 +#define WALLET_RPC_VERSION_MINOR 14 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -64,10 +64,12 @@ namespace wallet_rpc uint32_t account_index; std::set<uint32_t> address_indices; bool all_accounts; + bool strict; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) KV_SERIALIZE(address_indices) KV_SERIALIZE_OPT(all_accounts, false); + KV_SERIALIZE_OPT(strict, false); END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -230,9 +232,11 @@ namespace wallet_rpc struct request_t { std::string tag; // all accounts if empty, otherwise those accounts with this tag + bool strict_balances; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tag) + KV_SERIALIZE_OPT(strict_balances, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -2418,9 +2422,11 @@ namespace wallet_rpc struct response_t { uint32_t version; + bool release; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(version) + KV_SERIALIZE(release) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; diff --git a/tests/functional_tests/transactions_flow_test.cpp b/tests/functional_tests/transactions_flow_test.cpp index 32b601d7a..2add66b8b 100644 --- a/tests/functional_tests/transactions_flow_test.cpp +++ b/tests/functional_tests/transactions_flow_test.cpp @@ -173,7 +173,7 @@ bool transactions_flow_test(std::string& working_folder, //wait for money, until balance will have enough money w1.refresh(true, blocks_fetched, received_money, ok); - while(w1.unlocked_balance(0) < amount_to_transfer) + while(w1.unlocked_balance(0, true) < amount_to_transfer) { misc_utils::sleep_no_w(1000); w1.refresh(true, blocks_fetched, received_money, ok); @@ -186,7 +186,7 @@ bool transactions_flow_test(std::string& working_folder, { tools::wallet2::transfer_container incoming_transfers; w1.get_transfers(incoming_transfers); - if(incoming_transfers.size() > FIRST_N_TRANSFERS && get_money_in_first_transfers(incoming_transfers, FIRST_N_TRANSFERS) < w1.unlocked_balance(0) ) + if(incoming_transfers.size() > FIRST_N_TRANSFERS && get_money_in_first_transfers(incoming_transfers, FIRST_N_TRANSFERS) < w1.unlocked_balance(0, true) ) { //lets go! size_t count = 0; @@ -221,7 +221,7 @@ bool transactions_flow_test(std::string& working_folder, for(i = 0; i != transactions_count; i++) { uint64_t amount_to_tx = (amount_to_transfer - transfered_money) > transfer_size ? transfer_size: (amount_to_transfer - transfered_money); - while(w1.unlocked_balance(0) < amount_to_tx + TEST_FEE) + while(w1.unlocked_balance(0, true) < amount_to_tx + TEST_FEE) { misc_utils::sleep_no_w(1000); LOG_PRINT_L0("not enough money, waiting for cashback or mining"); @@ -270,7 +270,7 @@ bool transactions_flow_test(std::string& working_folder, misc_utils::sleep_no_w(DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN*1000);//wait two blocks before sync on another wallet on another daemon } - uint64_t money_2 = w2.balance(0); + uint64_t money_2 = w2.balance(0, true); if(money_2 == transfered_money) { MGINFO_GREEN("-----------------------FINISHING TRANSACTIONS FLOW TEST OK-----------------------"); diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index fe9ef418e..6c16a0a85 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -149,11 +149,11 @@ namespace } // Implement epee::net_utils::i_service_endpoint interface - virtual bool do_send(const void* ptr, size_t cb) + virtual bool do_send(epee::byte_slice message) { m_send_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); - m_last_send_data.append(reinterpret_cast<const char*>(ptr), cb); + m_last_send_data.append(reinterpret_cast<const char*>(message.data()), message.size()); return m_send_return; } diff --git a/tests/performance_tests/check_hash.h b/tests/performance_tests/check_hash.h index 53746fec4..294654f37 100644 --- a/tests/performance_tests/check_hash.h +++ b/tests/performance_tests/check_hash.h @@ -29,6 +29,7 @@ #pragma once #include "string_tools.h" +#include "int-util.h" #include "cryptonote_basic/difficulty.h" template<uint64_t hash_target_high, uint64_t hash_target_low, uint64_t difficulty_high, uint64_t difficulty_low> @@ -44,13 +45,18 @@ public: difficulty = difficulty_high; difficulty = (difficulty << 64) | difficulty_low; boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; - ((uint64_t*)&hash)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + uint64_t val; + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[0] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&hash)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[1] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&hash)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[2] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&hash)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[3] = SWAP64LE(val); return true; } diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index c32e0df20..bd7414c59 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -57,7 +57,6 @@ #include "rct_mlsag.h" #include "equality.h" #include "range_proof.h" -#include "rct_mlsag.h" #include "bulletproof.h" #include "crypto_ops.h" #include "multiexp.h" @@ -214,14 +213,8 @@ int main(int argc, char** argv) TEST_PERFORMANCE1(filter, p, test_cn_fast_hash, 32); TEST_PERFORMANCE1(filter, p, test_cn_fast_hash, 16384); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 3, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 5, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 10, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 100, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 3, true); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 5, true); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 10, true); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 100, true); + TEST_PERFORMANCE2(filter, p, test_ringct_mlsag, 11, false); + TEST_PERFORMANCE2(filter, p, test_ringct_mlsag, 11, true); TEST_PERFORMANCE2(filter, p, test_equality, memcmp32, true); TEST_PERFORMANCE2(filter, p, test_equality, memcmp32, false); @@ -251,15 +244,6 @@ int main(int argc, char** argv) TEST_PERFORMANCE6(filter, p, test_aggregated_bulletproof, false, 2, 1, 1, 0, 64); TEST_PERFORMANCE6(filter, p, test_aggregated_bulletproof, true, 2, 1, 1, 0, 64); // 64 proof, each with 2 amounts - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 3, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 5, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 10, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 100, false); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 3, true); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 5, true); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 10, true); - TEST_PERFORMANCE3(filter, p, test_ringct_mlsag, 1, 100, true); - TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_sc_add); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_sc_sub); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_sc_mul); diff --git a/tests/performance_tests/rct_mlsag.h b/tests/performance_tests/rct_mlsag.h index 0141710f7..59eae074e 100644 --- a/tests/performance_tests/rct_mlsag.h +++ b/tests/performance_tests/rct_mlsag.h @@ -35,13 +35,13 @@ #include "single_tx_test_base.h" -template<size_t inputs, size_t ring_size, bool ver> +template<size_t ring_size, bool ver> class test_ringct_mlsag : public single_tx_test_base { public: static const size_t cols = ring_size; - static const size_t rows = inputs; - static const size_t loop_count = 100; + static const size_t rows = 2; // single spend and commitment data + static const size_t loop_count = 1000; bool init() { @@ -65,7 +65,7 @@ public: { sk[j] = xm[ind][j]; } - IIccss = MLSAG_Gen(rct::identity(), P, sk, NULL, NULL, ind, rows, hw::get_device("default")); + IIccss = MLSAG_Gen(rct::identity(), P, sk, NULL, NULL, ind, rows-1, hw::get_device("default")); return true; } @@ -73,9 +73,9 @@ public: bool test() { if (ver) - MLSAG_Ver(rct::identity(), P, IIccss, rows); + MLSAG_Ver(rct::identity(), P, IIccss, rows-1); else - MLSAG_Gen(rct::identity(), P, sk, NULL, NULL, ind, rows, hw::get_device("default")); + MLSAG_Gen(rct::identity(), P, sk, NULL, NULL, ind, rows-1, hw::get_device("default")); return true; } diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 1c4c4384c..a5d179040 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -56,6 +56,7 @@ set(unit_tests_sources hmac_keccak.cpp http.cpp keccak.cpp + levin.cpp logging.cpp long_term_block_weight.cpp lmdb.cpp diff --git a/tests/unit_tests/difficulty.cpp b/tests/unit_tests/difficulty.cpp index a732e6969..e9e3272f0 100644 --- a/tests/unit_tests/difficulty.cpp +++ b/tests/unit_tests/difficulty.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "gtest/gtest.h" +#include "int-util.h" #include "cryptonote_basic/difficulty.h" static cryptonote::difficulty_type MKDIFF(uint64_t high, uint64_t low) @@ -42,13 +43,18 @@ static crypto::hash MKHASH(uint64_t high, uint64_t low) hash_target = (hash_target << 64) | low; boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; crypto::hash h; - ((uint64_t*)&h)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + uint64_t val; + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[0] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&h)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[1] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&h)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[2] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&h)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[3] = SWAP64LE(val); return h; } diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index 697845f60..b27699a77 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -140,12 +140,12 @@ namespace } // Implement epee::net_utils::i_service_endpoint interface - virtual bool do_send(const void* ptr, size_t cb) + virtual bool do_send(epee::byte_slice message) { //std::cout << "test_connection::do_send()" << std::endl; m_send_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); - m_last_send_data.append(reinterpret_cast<const char*>(ptr), cb); + m_last_send_data.append(reinterpret_cast<const char*>(message.data()), message.size()); return m_send_return; } @@ -241,13 +241,13 @@ namespace m_in_data.assign(256, 't'); - m_req_head.m_signature = LEVIN_SIGNATURE; - m_req_head.m_cb = m_in_data.size(); + m_req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + m_req_head.m_cb = SWAP64LE(m_in_data.size()); m_req_head.m_have_to_return_data = true; - m_req_head.m_command = expected_command; - m_req_head.m_return_code = LEVIN_OK; - m_req_head.m_flags = LEVIN_PACKET_REQUEST; - m_req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + m_req_head.m_command = SWAP32LE(expected_command); + m_req_head.m_return_code = SWAP32LE(LEVIN_OK); + m_req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + m_req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); m_commands_handler.return_code(expected_return_code); m_commands_handler.invoke_out_buf(m_expected_invoke_out_buf); @@ -337,12 +337,12 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process std::string in_data(256, 'q'); epee::levin::bucket_head2 req_head; - req_head.m_signature = LEVIN_SIGNATURE; - req_head.m_cb = in_data.size(); + req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + req_head.m_cb = SWAP64LE(in_data.size()); req_head.m_have_to_return_data = true; - req_head.m_command = expected_command; - req_head.m_flags = LEVIN_PACKET_REQUEST; - req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + req_head.m_command = SWAP32LE(expected_command); + req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); std::string buf(reinterpret_cast<const char*>(&req_head), sizeof(req_head)); buf += in_data; @@ -367,19 +367,19 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process // Parse send data std::string send_data = conn->last_send_data(); epee::levin::bucket_head2 resp_head; - resp_head = *reinterpret_cast<const epee::levin::bucket_head2*>(send_data.data()); ASSERT_LT(sizeof(resp_head), send_data.size()); + std::memcpy(std::addressof(resp_head), send_data.data(), sizeof(resp_head)); std::string out_data = send_data.substr(sizeof(resp_head)); // Check sent response ASSERT_EQ(expected_out_data, out_data); - ASSERT_EQ(LEVIN_SIGNATURE, resp_head.m_signature); - ASSERT_EQ(expected_command, resp_head.m_command); - ASSERT_EQ(expected_return_code, resp_head.m_return_code); - ASSERT_EQ(expected_out_data.size(), resp_head.m_cb); + ASSERT_EQ(LEVIN_SIGNATURE, SWAP64LE(resp_head.m_signature)); + ASSERT_EQ(expected_command, SWAP32LE(resp_head.m_command)); + ASSERT_EQ(expected_return_code, SWAP32LE(resp_head.m_return_code)); + ASSERT_EQ(expected_out_data.size(), SWAP64LE(resp_head.m_cb)); ASSERT_FALSE(resp_head.m_have_to_return_data); - ASSERT_EQ(LEVIN_PROTOCOL_VER_1, resp_head.m_protocol_version); - ASSERT_TRUE(0 != (resp_head.m_flags & LEVIN_PACKET_RESPONSE)); + ASSERT_EQ(SWAP32LE(LEVIN_PROTOCOL_VER_1), resp_head.m_protocol_version); + ASSERT_TRUE(0 != (SWAP32LE(resp_head.m_flags) & LEVIN_PACKET_RESPONSE)); } TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_processes_handle_read_as_notify) @@ -392,12 +392,12 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process std::string in_data(256, 'e'); epee::levin::bucket_head2 req_head; - req_head.m_signature = LEVIN_SIGNATURE; - req_head.m_cb = in_data.size(); + req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + req_head.m_cb = SWAP64LE(in_data.size()); req_head.m_have_to_return_data = false; - req_head.m_command = expected_command; - req_head.m_flags = LEVIN_PACKET_REQUEST; - req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + req_head.m_command = SWAP32LE(expected_command); + req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); std::string buf(reinterpret_cast<const char*>(&req_head), sizeof(req_head)); buf += in_data; @@ -425,6 +425,95 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process ASSERT_EQ(3, m_commands_handler.callback_counter()); } +TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_processes_handle_read_as_dummy) +{ + // Setup + const int expected_command = 4673261; + const std::string in_data(256, 'e'); + + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); + const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); + + test_connection_ptr conn = create_connection(); + + // Test + ASSERT_TRUE(conn->m_protocol_handler.handle_recv(noise.data(), noise.size())); + + // Check connection and levin_commands_handler states + ASSERT_EQ(0u, m_commands_handler.notify_counter()); + ASSERT_EQ(0u, m_commands_handler.invoke_counter()); + ASSERT_EQ(-1, m_commands_handler.last_command()); + ASSERT_TRUE(m_commands_handler.last_in_buf().empty()); + ASSERT_EQ(0u, conn->send_counter()); + ASSERT_TRUE(conn->last_send_data().empty()); + + + ASSERT_TRUE(conn->m_protocol_handler.handle_recv(notify.data(), notify.size())); + + // Check connection and levin_commands_handler states + ASSERT_EQ(1u, m_commands_handler.notify_counter()); + ASSERT_EQ(0u, m_commands_handler.invoke_counter()); + ASSERT_EQ(expected_command, m_commands_handler.last_command()); + ASSERT_EQ(in_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(0u, conn->send_counter()); + ASSERT_TRUE(conn->last_send_data().empty()); +} + +TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_processes_handle_read_as_fragment) +{ + // Setup + const int expected_command = 4673261; + const int expected_fragmented_command = 46732; + const std::string in_data(256, 'e'); + std::string in_fragmented_data(1024 * 4, 'c'); + + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); + const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); + epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise, expected_fragmented_command, epee::strspan<std::uint8_t>(in_fragmented_data)); + + EXPECT_EQ(5u, fragmented.size() / 1024); + EXPECT_EQ(0u, fragmented.size() % 1024); + + test_connection_ptr conn = create_connection(); + + while (!fragmented.empty()) + { + if ((fragmented.size() / 1024) % 2 == 1) + { + ASSERT_TRUE(conn->m_protocol_handler.handle_recv(notify.data(), notify.size())); + } + + ASSERT_EQ(3u - (fragmented.size() / 2048), m_commands_handler.notify_counter()); + ASSERT_EQ(0u, m_commands_handler.invoke_counter()); + ASSERT_EQ(expected_command, m_commands_handler.last_command()); + ASSERT_EQ(in_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(0u, conn->send_counter()); + ASSERT_TRUE(conn->last_send_data().empty()); + + epee::byte_slice next = fragmented.take_slice(1024); + ASSERT_TRUE(conn->m_protocol_handler.handle_recv(next.data(), next.size())); + } + + in_fragmented_data.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + ASSERT_EQ(4u, m_commands_handler.notify_counter()); + ASSERT_EQ(0u, m_commands_handler.invoke_counter()); + ASSERT_EQ(expected_fragmented_command, m_commands_handler.last_command()); + ASSERT_EQ(in_fragmented_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(0u, conn->send_counter()); + ASSERT_TRUE(conn->last_send_data().empty()); + + + ASSERT_TRUE(conn->m_protocol_handler.handle_recv(notify.data(), notify.size())); + + ASSERT_EQ(5u, m_commands_handler.notify_counter()); + ASSERT_EQ(0u, m_commands_handler.invoke_counter()); + ASSERT_EQ(expected_command, m_commands_handler.last_command()); + ASSERT_EQ(in_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(0u, conn->send_counter()); + ASSERT_TRUE(conn->last_send_data().empty()); +} + + TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_big_packet_1) { std::string buf("yyyyyy"); @@ -529,7 +618,23 @@ TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_two_re TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_unexpected_response) { - m_req_head.m_flags = LEVIN_PACKET_RESPONSE; + m_req_head.m_flags = SWAP32LE(LEVIN_PACKET_RESPONSE); + prepare_buf(); + + ASSERT_FALSE(m_conn->m_protocol_handler.handle_recv(m_buf.data(), m_buf.size())); +} + +TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_short_fragment) +{ + m_req_head.m_cb = 1; + m_req_head.m_flags = LEVIN_PACKET_BEGIN; + m_req_head.m_command = 0; + m_in_data.resize(1); + prepare_buf(); + + ASSERT_TRUE(m_conn->m_protocol_handler.handle_recv(m_buf.data(), m_buf.size())); + + m_req_head.m_flags = LEVIN_PACKET_END; prepare_buf(); ASSERT_FALSE(m_conn->m_protocol_handler.handle_recv(m_buf.data(), m_buf.size())); diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 32328edd9..6f887afda 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -44,6 +44,7 @@ #include "boost/archive/portable_binary_iarchive.hpp" #include "boost/archive/portable_binary_oarchive.hpp" +#include "byte_slice.h" #include "hex.h" #include "net/net_utils_base.h" #include "net/local_ip.h" @@ -375,6 +376,438 @@ TEST(Span, ToMutSpan) EXPECT_EQ((std::vector<unsigned>{1, 2, 3, 4}), mut); } +TEST(ByteSlice, Construction) +{ + EXPECT_TRUE(std::is_default_constructible<epee::byte_slice>()); + EXPECT_TRUE(std::is_move_constructible<epee::byte_slice>()); + EXPECT_FALSE(std::is_copy_constructible<epee::byte_slice>()); + EXPECT_TRUE(std::is_move_assignable<epee::byte_slice>()); + EXPECT_FALSE(std::is_copy_assignable<epee::byte_slice>()); +} + +TEST(ByteSlice, NoExcept) +{ + EXPECT_TRUE(std::is_nothrow_default_constructible<epee::byte_slice>()); + EXPECT_TRUE(std::is_nothrow_move_constructible<epee::byte_slice>()); + EXPECT_TRUE(std::is_nothrow_move_assignable<epee::byte_slice>()); + + epee::byte_slice lvalue{}; + const epee::byte_slice clvalue{}; + + EXPECT_TRUE(noexcept(lvalue.clone())); + EXPECT_TRUE(noexcept(clvalue.clone())); + + EXPECT_TRUE(noexcept(lvalue.begin())); + EXPECT_TRUE(noexcept(clvalue.begin())); + EXPECT_TRUE(noexcept(lvalue.end())); + EXPECT_TRUE(noexcept(clvalue.end())); + + EXPECT_TRUE(noexcept(lvalue.cbegin())); + EXPECT_TRUE(noexcept(clvalue.cbegin())); + EXPECT_TRUE(noexcept(lvalue.cend())); + EXPECT_TRUE(noexcept(clvalue.cend())); + + EXPECT_TRUE(noexcept(lvalue.empty())); + EXPECT_TRUE(noexcept(clvalue.empty())); + + EXPECT_TRUE(noexcept(lvalue.data())); + EXPECT_TRUE(noexcept(clvalue.data())); + EXPECT_TRUE(noexcept(lvalue.size())); + EXPECT_TRUE(noexcept(clvalue.size())); + + EXPECT_TRUE(noexcept(lvalue.remove_prefix(0))); + EXPECT_TRUE(noexcept(lvalue.take_slice(0))); +} + +TEST(ByteSlice, Empty) +{ + epee::byte_slice slice{}; + + EXPECT_EQ(slice.begin(), slice.end()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_TRUE(slice.empty()); + EXPECT_EQ(0u, slice.size()); + EXPECT_EQ(slice.begin(), slice.data()); + + EXPECT_EQ(0u, slice.get_slice(0, 0).size()); + EXPECT_THROW(slice.get_slice(0, 1), std::out_of_range); + EXPECT_EQ(0u, slice.remove_prefix(1)); + EXPECT_EQ(0u, slice.take_slice(1).size()); +} + +TEST(ByteSlice, CopySpans) +{ + const epee::span<const std::uint8_t> part1 = epee::as_byte_span("this is part1"); + const epee::span<const std::uint8_t> part2 = epee::as_byte_span("then part2"); + const epee::span<const std::uint8_t> part3 = epee::as_byte_span("finally part3"); + + const epee::byte_slice slice{part1, part2, part3}; + + EXPECT_NE(nullptr, slice.begin()); + EXPECT_NE(nullptr, slice.end()); + EXPECT_NE(slice.begin(), slice.end()); + EXPECT_NE(slice.cbegin(), slice.cend()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + ASSERT_EQ(slice.size(), std::size_t(slice.end() - slice.begin())); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.begin(), slice.data()); + ASSERT_EQ(part1.size() + part2.size() + part3.size(), slice.size()); + EXPECT_TRUE( + boost::range::equal( + part1, boost::make_iterator_range(slice.begin(), slice.begin() + part1.size()) + ) + ); + EXPECT_TRUE( + boost::range::equal( + part2, boost::make_iterator_range(slice.begin() + part1.size(), slice.end() - part3.size()) + ) + ); + EXPECT_TRUE( + boost::range::equal( + part3, boost::make_iterator_range(slice.end() - part3.size(), slice.end()) + ) + ); +} + +TEST(ByteSlice, AdaptString) +{ + static constexpr const char base_string[] = "this is an example message"; + std::string adapted = base_string; + + const epee::span<const uint8_t> original = epee::to_byte_span(epee::to_span(adapted)); + const epee::byte_slice slice{std::move(adapted)}; + + EXPECT_EQ(original.begin(), slice.begin()); + EXPECT_EQ(original.cbegin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(original.cend(), slice.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(original.data(), slice.data()); + EXPECT_EQ(original.size(), slice.size()); + EXPECT_TRUE(boost::range::equal(boost::string_ref{base_string}, slice)); +} + +TEST(ByteSlice, EmptyAdaptString) +{ + epee::byte_slice slice{std::string{}}; + + EXPECT_EQ(slice.begin(), slice.end()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_TRUE(slice.empty()); + EXPECT_EQ(0u, slice.size()); + EXPECT_EQ(slice.begin(), slice.data()); + + EXPECT_EQ(0u, slice.get_slice(0, 0).size()); + EXPECT_THROW(slice.get_slice(0, 1), std::out_of_range); + EXPECT_EQ(0u, slice.remove_prefix(1)); + EXPECT_EQ(0u, slice.take_slice(1).size()); +} + +TEST(ByteSlice, AdaptVector) +{ + static constexpr const char base_string[] = "this is an example message"; + std::vector<std::uint8_t> adapted(sizeof(base_string)); + + ASSERT_EQ(sizeof(base_string), adapted.size()); + std::memcpy(adapted.data(), base_string, sizeof(base_string)); + + const epee::span<const uint8_t> original = epee::to_span(adapted); + const epee::byte_slice slice{std::move(adapted)}; + + EXPECT_EQ(sizeof(base_string), original.size()); + + EXPECT_EQ(original.begin(), slice.begin()); + EXPECT_EQ(original.cbegin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(original.cend(), slice.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(original.data(), slice.data()); + EXPECT_EQ(original.size(), slice.size()); + EXPECT_TRUE(boost::range::equal(base_string, slice)); +} + +TEST(ByteSlice, EmptyAdaptVector) +{ + epee::byte_slice slice{std::vector<std::uint8_t>{}}; + + EXPECT_EQ(slice.begin(), slice.end()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_TRUE(slice.empty()); + EXPECT_EQ(0u, slice.size()); + EXPECT_EQ(slice.begin(), slice.data()); + + EXPECT_EQ(0u, slice.get_slice(0, 0).size()); + EXPECT_THROW(slice.get_slice(0, 1), std::out_of_range); + EXPECT_EQ(0u, slice.remove_prefix(1)); + EXPECT_EQ(0u, slice.take_slice(1).size()); +} + +TEST(ByteSlice, Move) +{ + static constexpr const char base_string[] = "another example message"; + + epee::byte_slice slice{epee::as_byte_span(base_string)}; + EXPECT_TRUE(boost::range::equal(base_string, slice)); + + const epee::span<const std::uint8_t> original = epee::to_span(slice); + epee::byte_slice moved{std::move(slice)}; + EXPECT_TRUE(boost::range::equal(base_string, moved)); + + EXPECT_EQ(slice.begin(), slice.end()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_EQ(original.begin(), moved.begin()); + EXPECT_EQ(moved.begin(), moved.cbegin()); + EXPECT_EQ(original.end(), moved.end()); + EXPECT_EQ(moved.end(), moved.cend()); + + EXPECT_TRUE(slice.empty()); + EXPECT_EQ(slice.begin(), slice.data()); + EXPECT_EQ(0u, slice.size()); + + EXPECT_FALSE(moved.empty()); + EXPECT_EQ(moved.begin(), moved.data()); + EXPECT_EQ(original.size(), moved.size()); + + slice = std::move(moved); + EXPECT_TRUE(boost::range::equal(base_string, slice)); + + EXPECT_EQ(original.begin(), slice.begin()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.begin(), slice.data()); + EXPECT_EQ(original.size(), slice.size()); + + EXPECT_TRUE(moved.empty()); + EXPECT_EQ(moved.begin(), moved.data()); + EXPECT_EQ(0u, moved.size()); +} + +TEST(ByteSlice, Clone) +{ + static constexpr const char base_string[] = "another example message"; + + const epee::byte_slice slice{epee::as_byte_span(base_string)}; + EXPECT_TRUE(boost::range::equal(base_string, slice)); + + const epee::byte_slice clone{slice.clone()}; + EXPECT_TRUE(boost::range::equal(base_string, clone)); + + EXPECT_EQ(slice.begin(), clone.begin()); + EXPECT_EQ(slice.cbegin(), clone.cbegin()); + EXPECT_EQ(slice.end(), clone.end()); + EXPECT_EQ(slice.cend(), clone.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_FALSE(clone.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(slice.data(), clone.data()); + EXPECT_EQ(sizeof(base_string), slice.size()); + EXPECT_EQ(slice.size(), clone.size()); +} + +TEST(ByteSlice, RemovePrefix) +{ + static constexpr const char base_string[] = "another example message"; + static constexpr std::size_t remove_size = sizeof("another"); + static constexpr std::size_t remaining = sizeof(base_string) - remove_size; + + epee::byte_slice slice{epee::as_byte_span(base_string)}; + EXPECT_TRUE(boost::range::equal(base_string, slice)); + + const epee::span<const std::uint8_t> original = epee::to_span(slice); + EXPECT_EQ(remove_size, slice.remove_prefix(remove_size)); + + EXPECT_EQ(original.begin() + remove_size, slice.begin()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(remaining, slice.size()); + + // touch original pointers to check "free" status + EXPECT_TRUE(boost::range::equal(base_string, original)); + + EXPECT_EQ(remaining, slice.remove_prefix(remaining + 1)); + + EXPECT_EQ(slice.begin(), slice.end()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_TRUE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(0, slice.size()); +} + +TEST(ByteSlice, TakeSlice) +{ + static constexpr const char base_string[] = "another example message"; + static constexpr std::size_t remove_size = sizeof("another"); + static constexpr std::size_t remaining = sizeof(base_string) - remove_size; + + epee::byte_slice slice{epee::as_byte_span(base_string)}; + EXPECT_TRUE(boost::range::equal(base_string, slice)); + + const epee::span<const std::uint8_t> original = epee::to_span(slice); + const epee::byte_slice slice2 = slice.take_slice(remove_size); + + EXPECT_EQ(original.begin() + remove_size, slice.begin()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_EQ(original.begin(), slice2.begin()); + EXPECT_EQ(slice2.begin(), slice2.cbegin()); + EXPECT_EQ(original.begin() + remove_size, slice2.end()); + EXPECT_EQ(slice2.end(), slice2.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(remaining, slice.size()); + + EXPECT_FALSE(slice2.empty()); + EXPECT_EQ(slice2.cbegin(), slice2.data()); + EXPECT_EQ(remove_size, slice2.size()); + + // touch original pointers to check "free" status + EXPECT_TRUE(boost::range::equal(base_string, original)); + + const epee::byte_slice slice3 = slice.take_slice(remaining + 1); + + EXPECT_EQ(slice.begin(), slice.end()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_EQ(original.begin(), slice2.begin()); + EXPECT_EQ(slice2.begin(), slice2.cbegin()); + EXPECT_EQ(original.begin() + remove_size, slice2.end()); + EXPECT_EQ(slice2.end(), slice2.cend()); + + EXPECT_EQ(slice2.end(), slice3.begin()); + EXPECT_EQ(slice3.begin(), slice3.cbegin()); + EXPECT_EQ(original.end(), slice3.end()); + EXPECT_EQ(slice3.end(), slice3.cend()); + + EXPECT_TRUE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(0, slice.size()); + + EXPECT_FALSE(slice2.empty()); + EXPECT_EQ(slice2.cbegin(), slice2.data()); + EXPECT_EQ(remove_size, slice2.size()); + + EXPECT_FALSE(slice3.empty()); + EXPECT_EQ(slice3.cbegin(), slice3.data()); + EXPECT_EQ(remaining, slice3.size()); + + // touch original pointers to check "free" status + slice = nullptr; + EXPECT_TRUE(boost::range::equal(base_string, original)); +} + +TEST(ByteSlice, GetSlice) +{ + static constexpr const char base_string[] = "another example message"; + static constexpr std::size_t get_size = sizeof("another"); + static constexpr std::size_t get2_size = sizeof(base_string) - get_size; + + epee::span<const std::uint8_t> original{}; + epee::byte_slice slice2{}; + epee::byte_slice slice3{}; + + // make sure get_slice increments ref count + { + const epee::byte_slice slice{epee::as_byte_span(base_string)}; + EXPECT_TRUE(boost::range::equal(base_string, slice)); + + original = epee::to_span(slice); + slice2 = slice.get_slice(0, get_size); + + EXPECT_EQ(original.begin(), slice.begin()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_EQ(original.begin(), slice2.begin()); + EXPECT_EQ(slice2.begin(), slice2.cbegin()); + EXPECT_EQ(original.begin() + get_size, slice2.end()); + EXPECT_EQ(slice2.end(), slice2.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(original.size(), slice.size()); + + EXPECT_FALSE(slice2.empty()); + EXPECT_EQ(slice2.cbegin(), slice2.data()); + EXPECT_EQ(get_size, slice2.size()); + + // touch original pointers to check "free" status + EXPECT_TRUE(boost::range::equal(base_string, original)); + + slice3 = slice.get_slice(get_size, sizeof(base_string)); + + EXPECT_EQ(original.begin(), slice.begin()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_EQ(original.begin(), slice2.begin()); + EXPECT_EQ(slice2.begin(), slice2.cbegin()); + EXPECT_EQ(original.begin() + get_size, slice2.end()); + EXPECT_EQ(slice2.end(), slice2.cend()); + + EXPECT_EQ(slice2.end(), slice3.begin()); + EXPECT_EQ(slice3.begin(), slice3.cbegin()); + EXPECT_EQ(original.end(), slice3.end()); + EXPECT_EQ(slice3.end(), slice3.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(original.size(), slice.size()); + + EXPECT_FALSE(slice2.empty()); + EXPECT_EQ(slice2.cbegin(), slice2.data()); + EXPECT_EQ(get_size, slice2.size()); + + EXPECT_FALSE(slice3.empty()); + EXPECT_EQ(slice3.cbegin(), slice3.data()); + EXPECT_EQ(get2_size, slice3.size()); + + EXPECT_THROW(slice.get_slice(1, 0), std::out_of_range); + EXPECT_THROW(slice.get_slice(0, sizeof(base_string) + 1), std::out_of_range); + EXPECT_THROW(slice.get_slice(sizeof(base_string) + 1, sizeof(base_string) + 1), std::out_of_range); + EXPECT_TRUE(slice.get_slice(sizeof(base_string), sizeof(base_string)).empty()); + + EXPECT_EQ(original.begin(), slice.begin()); + EXPECT_EQ(slice.begin(), slice.cbegin()); + EXPECT_EQ(original.end(), slice.end()); + EXPECT_EQ(slice.end(), slice.cend()); + + EXPECT_FALSE(slice.empty()); + EXPECT_EQ(slice.cbegin(), slice.data()); + EXPECT_EQ(original.size(), slice.size()); + } + + // touch original pointers to check "free" status + EXPECT_TRUE(boost::range::equal(base_string, original)); +} + TEST(ToHex, String) { EXPECT_TRUE(epee::to_hex::string(nullptr).empty()); diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp new file mode 100644 index 000000000..3188167f9 --- /dev/null +++ b/tests/unit_tests/levin.cpp @@ -0,0 +1,586 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <algorithm> +#include <boost/uuid/nil_generator.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/uuid/uuid.hpp> +#include <cstring> +#include <gtest/gtest.h> +#include <limits> +#include <set> + +#include "byte_slice.h" +#include "crypto/crypto.h" +#include "cryptonote_basic/connection_context.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "cryptonote_protocol/levin_notify.h" +#include "int-util.h" +#include "p2p/net_node.h" +#include "net/dandelionpp.h" +#include "net/levin_base.h" +#include "span.h" + +namespace +{ + class test_endpoint final : public epee::net_utils::i_service_endpoint + { + boost::asio::io_service& io_service_; + std::size_t ref_count_; + + virtual bool do_send(epee::byte_slice message) override final + { + send_queue_.push_back(std::move(message)); + return true; + } + + virtual bool close() override final + { + return true; + } + + virtual bool send_done() override final + { + throw std::logic_error{"send_done not implemented"}; + } + + virtual bool call_run_once_service_io() override final + { + return io_service_.run_one(); + } + + virtual bool request_callback() override final + { + throw std::logic_error{"request_callback not implemented"}; + } + + virtual boost::asio::io_service& get_io_service() override final + { + return io_service_; + } + + virtual bool add_ref() override final + { + ++ref_count_; + return true; + } + + virtual bool release() override final + { + --ref_count_; + return true; + } + + public: + test_endpoint(boost::asio::io_service& io_service) + : epee::net_utils::i_service_endpoint(), + io_service_(io_service), + ref_count_(0), + send_queue_() + {} + + virtual ~test_endpoint() noexcept(false) override final + { + EXPECT_EQ(0u, ref_count_); + } + + std::deque<epee::byte_slice> send_queue_; + }; + + class test_connection + { + test_endpoint endpoint_; + cryptonote::levin::detail::p2p_context context_; + epee::levin::async_protocol_handler<cryptonote::levin::detail::p2p_context> handler_; + + public: + test_connection(boost::asio::io_service& io_service, cryptonote::levin::connections& connections, boost::uuids::random_generator& random_generator) + : context_(), + endpoint_(io_service), + handler_(std::addressof(endpoint_), connections, context_) + { + const_cast<boost::uuids::uuid&>(context_.m_connection_id) = random_generator(); + handler_.after_init_connection(); + } + + //\return Number of messages processed + std::size_t process_send_queue() + { + std::size_t count = 0; + for ( ; !endpoint_.send_queue_.empty(); ++count, endpoint_.send_queue_.pop_front()) + { + // invalid messages shoudn't be possible in this test; + EXPECT_TRUE(handler_.handle_recv(endpoint_.send_queue_.front().data(), endpoint_.send_queue_.front().size())); + } + return count; + } + + const boost::uuids::uuid& get_id() const noexcept + { + return context_.m_connection_id; + } + }; + + struct received_message + { + boost::uuids::uuid connection; + int command; + std::string payload; + }; + + class test_receiver : public epee::levin::levin_commands_handler<cryptonote::levin::detail::p2p_context> + { + std::deque<received_message> invoked_; + std::deque<received_message> notified_; + + template<typename T> + static std::pair<boost::uuids::uuid, typename T::request> get_message(std::deque<received_message>& queue) + { + if (queue.empty()) + throw std::logic_error{"Queue has no received messges"}; + + if (queue.front().command != T::ID) + throw std::logic_error{"Unexpected ID at front of message queue"}; + + epee::serialization::portable_storage storage{}; + if(!storage.load_from_binary(epee::strspan<std::uint8_t>(queue.front().payload))) + throw std::logic_error{"Unable to parse epee binary format"}; + + typename T::request request{}; + if (!request.load(storage)) + throw std::logic_error{"Unable to load into expected request"}; + + boost::uuids::uuid connection = queue.front().connection; + queue.pop_front(); + return {connection, std::move(request)}; + } + + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, cryptonote::levin::detail::p2p_context& context) override final + { + buff_out.clear(); + invoked_.push_back( + {context.m_connection_id, command, std::string{reinterpret_cast<const char*>(in_buff.data()), in_buff.size()}} + ); + return 1; + } + + virtual int notify(int command, const epee::span<const uint8_t> in_buff, cryptonote::levin::detail::p2p_context& context) override final + { + notified_.push_back( + {context.m_connection_id, command, std::string{reinterpret_cast<const char*>(in_buff.data()), in_buff.size()}} + ); + return 1; + } + + virtual void callback(cryptonote::levin::detail::p2p_context& context) override final + {} + + virtual void on_connection_new(cryptonote::levin::detail::p2p_context&) override final + {} + + virtual void on_connection_close(cryptonote::levin::detail::p2p_context&) override final + {} + + public: + test_receiver() + : epee::levin::levin_commands_handler<cryptonote::levin::detail::p2p_context>(), + invoked_(), + notified_() + {} + + virtual ~test_receiver() noexcept override final{} + + std::size_t invoked_size() const noexcept + { + return invoked_.size(); + } + + std::size_t notified_size() const noexcept + { + return notified_.size(); + } + + template<typename T> + std::pair<boost::uuids::uuid, typename T::request> get_invoked() + { + return get_message<T>(invoked_); + } + + template<typename T> + std::pair<boost::uuids::uuid, typename T::request> get_notification() + { + return get_message<T>(notified_); + } + }; + + class levin_notify : public ::testing::Test + { + const std::shared_ptr<cryptonote::levin::connections> connections_; + std::set<boost::uuids::uuid> connection_ids_; + + public: + levin_notify() + : ::testing::Test(), + connections_(std::make_shared<cryptonote::levin::connections>()), + connection_ids_(), + random_generator_(), + io_service_(), + receiver_(), + contexts_() + { + connections_->set_handler(std::addressof(receiver_), nullptr); + } + + virtual void TearDown() override final + { + EXPECT_EQ(0u, receiver_.invoked_size()); + EXPECT_EQ(0u, receiver_.notified_size()); + } + + void add_connection() + { + contexts_.emplace_back(io_service_, *connections_, random_generator_); + EXPECT_TRUE(connection_ids_.emplace(contexts_.back().get_id()).second); + EXPECT_EQ(connection_ids_.size(), connections_->get_connections_count()); + } + + cryptonote::levin::notify make_notifier(const std::size_t noise_size) + { + epee::byte_slice noise = nullptr; + if (noise_size) + noise = epee::levin::make_noise_notify(noise_size); + return cryptonote::levin::notify{io_service_, connections_, std::move(noise)}; + } + + boost::uuids::random_generator random_generator_; + boost::asio::io_service io_service_; + test_receiver receiver_; + std::deque<test_connection> contexts_; + }; +} + +TEST(make_header, no_expect_return) +{ + static constexpr const std::size_t max_length = std::numeric_limits<std::size_t>::max(); + + const epee::levin::bucket_head2 header1 = epee::levin::make_header(1024, max_length, 5601, false); + EXPECT_EQ(SWAP64LE(LEVIN_SIGNATURE), header1.m_signature); + EXPECT_FALSE(header1.m_have_to_return_data); + EXPECT_EQ(SWAP64LE(max_length), header1.m_cb); + EXPECT_EQ(SWAP32LE(1024), header1.m_command); + EXPECT_EQ(SWAP32LE(LEVIN_PROTOCOL_VER_1), header1.m_protocol_version); + EXPECT_EQ(SWAP32LE(5601), header1.m_flags); +} + +TEST(make_header, expect_return) +{ + const epee::levin::bucket_head2 header1 = epee::levin::make_header(65535, 0, 0, true); + EXPECT_EQ(SWAP64LE(LEVIN_SIGNATURE), header1.m_signature); + EXPECT_TRUE(header1.m_have_to_return_data); + EXPECT_EQ(0u, header1.m_cb); + EXPECT_EQ(SWAP32LE(65535), header1.m_command); + EXPECT_EQ(SWAP32LE(LEVIN_PROTOCOL_VER_1), header1.m_protocol_version); + EXPECT_EQ(0u, header1.m_flags); +} + +TEST(make_notify, empty_payload) +{ + const epee::byte_slice message = epee::levin::make_notify(443, nullptr); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, false); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(make_notify, with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + const epee::byte_slice message = epee::levin::make_notify(443, epee::strspan<std::uint8_t>(bytes)); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, false); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(make_noise, invalid) +{ + EXPECT_TRUE(epee::levin::make_noise_notify(sizeof(epee::levin::bucket_head2) - 1).empty()); +} + +TEST(make_noise, valid) +{ + static constexpr const std::uint32_t flags = + LEVIN_PACKET_BEGIN | LEVIN_PACKET_END; + + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); + const epee::levin::bucket_head2 header = + epee::levin::make_header(0, 1024 - sizeof(epee::levin::bucket_head2), flags, false); + + ASSERT_EQ(1024, noise.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), noise.data(), sizeof(header)) == 0); + EXPECT_EQ(1024 - sizeof(header), std::count(noise.cbegin() + sizeof(header), noise.cend(), 0)); +} + +TEST(make_fragment, invalid) +{ + EXPECT_TRUE(epee::levin::make_fragmented_notify(nullptr, 0, nullptr).empty()); +} + +TEST(make_fragment, single) +{ + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); + const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 11, nullptr); + const epee::levin::bucket_head2 header = + epee::levin::make_header(11, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_REQUEST, false); + + EXPECT_EQ(1024, noise.size()); + ASSERT_EQ(1024, fragment.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), fragment.data(), sizeof(header)) == 0); + EXPECT_EQ(1024 - sizeof(header), std::count(noise.cbegin() + sizeof(header), noise.cend(), 0)); +} + +TEST(make_fragment, multiple) +{ + std::string bytes(1024 * 3 - 150, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); + epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 114, epee::strspan<std::uint8_t>(bytes)); + + epee::levin::bucket_head2 header = + epee::levin::make_header(0, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_BEGIN, false); + + ASSERT_LE(sizeof(header), fragment.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), fragment.data(), sizeof(header)) == 0); + + fragment.take_slice(sizeof(header)); + header.m_flags = LEVIN_PACKET_REQUEST; + header.m_cb = bytes.size(); + header.m_command = 114; + + ASSERT_LE(sizeof(header), fragment.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), fragment.data(), sizeof(header)) == 0); + + fragment.take_slice(sizeof(header)); + + ASSERT_LE(bytes.size(), fragment.size()); + EXPECT_TRUE(std::memcmp(bytes.data(), fragment.data(), 1024 - sizeof(header) * 2) == 0); + + bytes.erase(0, 1024 - sizeof(header) * 2); + fragment.take_slice(1024 - sizeof(header) * 2); + header.m_flags = 0; + header.m_cb = 1024 - sizeof(header); + header.m_command = 0; + + ASSERT_LE(sizeof(header), fragment.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), fragment.data(), sizeof(header)) == 0); + + fragment.take_slice(sizeof(header)); + + ASSERT_LE(bytes.size(), fragment.size()); + EXPECT_TRUE(std::memcmp(bytes.data(), fragment.data(), 1024 - sizeof(header)) == 0); + + bytes.erase(0, 1024 - sizeof(header)); + fragment.take_slice(1024 - sizeof(header)); + header.m_flags = LEVIN_PACKET_END; + + ASSERT_LE(sizeof(header), fragment.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), fragment.data(), sizeof(header)) == 0); + + fragment.take_slice(sizeof(header)); + EXPECT_TRUE(std::memcmp(bytes.data(), fragment.data(), bytes.size()) == 0); + + fragment.take_slice(bytes.size()); + + EXPECT_EQ(18, std::count(fragment.cbegin(), fragment.cend(), 0)); +} + +TEST_F(levin_notify, defaulted) +{ + cryptonote::levin::notify notifier{}; + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + EXPECT_FALSE(notifier.send_txs({}, random_generator_(), false)); +} + +TEST_F(levin_notify, flood) +{ + cryptonote::levin::notify notifier = make_notifier(0); + + for (unsigned count = 0; count < 10; ++count) + add_connection(); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); // not tracked + } + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + txs[1].resize(200, 'f'); + + ASSERT_EQ(10u, contexts_.size()); + { + auto context = contexts_.begin(); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), false)); + + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + EXPECT_EQ(0u, context->process_send_queue()); + for (++context; context != contexts_.end(); ++context) + EXPECT_EQ(1u, context->process_send_queue()); + + ASSERT_EQ(9u, receiver_.notified_size()); + for (unsigned count = 0; count < 9; ++count) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_TRUE(notification._.empty()); + } + } + + ASSERT_EQ(10u, contexts_.size()); + { + auto context = contexts_.begin(); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), true)); + + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + EXPECT_EQ(0u, context->process_send_queue()); + for (++context; context != contexts_.end(); ++context) + EXPECT_EQ(1u, context->process_send_queue()); + + ASSERT_EQ(9u, receiver_.notified_size()); + for (unsigned count = 0; count < 9; ++count) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_FALSE(notification._.empty()); + } + } +} + +TEST_F(levin_notify, noise) +{ + for (unsigned count = 0; count < 10; ++count) + add_connection(); + + std::vector<cryptonote::blobdata> txs(1); + txs[0].resize(1900, 'h'); + + const boost::uuids::uuid incoming_id = random_generator_(); + cryptonote::levin::notify notifier = make_notifier(2048); + + { + const auto status = notifier.get_status(); + EXPECT_TRUE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + ASSERT_LT(0u, io_service_.poll()); + { + const auto status = notifier.get_status(); + EXPECT_TRUE(status.has_noise); + EXPECT_TRUE(status.connections_filled); + } + + notifier.run_stems(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + { + std::size_t sent = 0; + for (auto& context : contexts_) + sent += context.process_send_queue(); + + EXPECT_EQ(2u, sent); + EXPECT_EQ(0u, receiver_.notified_size()); + } + + EXPECT_TRUE(notifier.send_txs(txs, incoming_id, false)); + notifier.run_stems(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + { + std::size_t sent = 0; + for (auto& context : contexts_) + sent += context.process_send_queue(); + + ASSERT_EQ(2u, sent); + while (sent--) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_TRUE(notification._.empty()); + } + } + + txs[0].resize(3000, 'r'); + EXPECT_TRUE(notifier.send_txs(txs, incoming_id, true)); + notifier.run_stems(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + { + std::size_t sent = 0; + for (auto& context : contexts_) + sent += context.process_send_queue(); + + EXPECT_EQ(2u, sent); + EXPECT_EQ(0u, receiver_.notified_size()); + } + + notifier.run_stems(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + { + std::size_t sent = 0; + for (auto& context : contexts_) + sent += context.process_send_queue(); + + ASSERT_EQ(2u, sent); + while (sent--) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_TRUE(notification._.empty()); + } + } +} diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index 3acf75f3b..f24ffb45c 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -26,6 +26,7 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include <algorithm> #include <atomic> #include <boost/archive/portable_binary_oarchive.hpp> #include <boost/archive/portable_binary_iarchive.hpp> @@ -36,13 +37,22 @@ #include <boost/asio/steady_timer.hpp> #include <boost/asio/write.hpp> #include <boost/endian/conversion.hpp> +#include <boost/range/adaptor/sliced.hpp> +#include <boost/range/combine.hpp> #include <boost/system/error_code.hpp> #include <boost/thread/thread.hpp> +#include <boost/uuid/nil_generator.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/uuid/uuid.hpp> +#include <cstdint> #include <cstring> #include <functional> #include <gtest/gtest.h> +#include <map> #include <memory> +#include <type_traits> +#include "net/dandelionpp.h" #include "net/error.h" #include "net/net_utils_base.h" #include "net/socks.h" @@ -857,3 +867,389 @@ TEST(socks_connector, timeout) EXPECT_THROW(sock.get().is_open(), boost::system::system_error); } +TEST(dandelionpp_map, traits) +{ + EXPECT_TRUE(std::is_default_constructible<net::dandelionpp::connection_map>()); + EXPECT_TRUE(std::is_move_constructible<net::dandelionpp::connection_map>()); + EXPECT_TRUE(std::is_move_assignable<net::dandelionpp::connection_map>()); + EXPECT_FALSE(std::is_copy_constructible<net::dandelionpp::connection_map>()); + EXPECT_FALSE(std::is_copy_assignable<net::dandelionpp::connection_map>()); +} + +TEST(dandelionpp_map, empty) +{ + const net::dandelionpp::connection_map mapper{}; + + EXPECT_EQ(mapper.begin(), mapper.end()); + EXPECT_EQ(0u, mapper.size()); + + const net::dandelionpp::connection_map cloned = mapper.clone(); + EXPECT_EQ(cloned.begin(), cloned.end()); + EXPECT_EQ(0u, cloned.size()); +} + +TEST(dandelionpp_map, zero_stems) +{ + std::vector<boost::uuids::uuid> connections{6}; + std::generate(connections.begin(), connections.end(), boost::uuids::random_generator{}); + + net::dandelionpp::connection_map mapper{connections, 0}; + EXPECT_EQ(mapper.begin(), mapper.end()); + EXPECT_EQ(0u, mapper.size()); + + for (const boost::uuids::uuid& connection : connections) + EXPECT_TRUE(mapper.get_stem(connection).is_nil()); + + EXPECT_FALSE(mapper.update(connections)); + EXPECT_EQ(mapper.begin(), mapper.end()); + EXPECT_EQ(0u, mapper.size()); + + for (const boost::uuids::uuid& connection : connections) + EXPECT_TRUE(mapper.get_stem(connection).is_nil()); + + const net::dandelionpp::connection_map cloned = mapper.clone(); + EXPECT_EQ(cloned.end(), cloned.begin()); + EXPECT_EQ(0u, cloned.size()); +} + +TEST(dandelionpp_map, dropped_connection) +{ + std::vector<boost::uuids::uuid> connections{6}; + std::generate(connections.begin(), connections.end(), boost::uuids::random_generator{}); + std::sort(connections.begin(), connections.end()); + + // select 3 of 6 outgoing connections + net::dandelionpp::connection_map mapper{connections, 3}; + EXPECT_EQ(3u, mapper.size()); + EXPECT_EQ(3, mapper.end() - mapper.begin()); + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + { + const net::dandelionpp::connection_map cloned = mapper.clone(); + EXPECT_EQ(3u, cloned.size()); + ASSERT_EQ(mapper.end() - mapper.begin(), cloned.end() - cloned.begin()); + for (auto elem : boost::combine(mapper, cloned)) + EXPECT_EQ(boost::get<0>(elem), boost::get<1>(elem)); + } + EXPECT_FALSE(mapper.update(connections)); + EXPECT_EQ(3u, mapper.size()); + ASSERT_EQ(3, mapper.end() - mapper.begin()); + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + std::map<boost::uuids::uuid, boost::uuids::uuid> mapping; + std::vector<boost::uuids::uuid> in_connections{9}; + std::generate(in_connections.begin(), in_connections.end(), boost::uuids::random_generator{}); + { + std::map<boost::uuids::uuid, std::size_t> used; + std::multimap<boost::uuids::uuid, boost::uuids::uuid> inverse_mapping; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + EXPECT_TRUE(mapping.emplace(connection, out).second); + inverse_mapping.emplace(out, connection); + used[out]++; + } + + EXPECT_EQ(3u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(3u, entry.second); + + for (const boost::uuids::uuid& connection : in_connections) + EXPECT_EQ(mapping[connection], mapper.get_stem(connection)); + + // drop 1 connection, and select replacement from 1 of unused 3. + const boost::uuids::uuid lost_connection = *(++mapper.begin()); + const auto elem = std::lower_bound(connections.begin(), connections.end(), lost_connection); + ASSERT_NE(connections.end(), elem); + ASSERT_EQ(lost_connection, *elem); + connections.erase(elem); + + EXPECT_TRUE(mapper.update(connections)); + EXPECT_EQ(3u, mapper.size()); + ASSERT_EQ(3, mapper.end() - mapper.begin()); + + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_NE(lost_connection, connection); + } + + const boost::uuids::uuid newly_mapped = *(++mapper.begin()); + EXPECT_FALSE(newly_mapped.is_nil()); + EXPECT_NE(lost_connection, newly_mapped); + + for (auto elems = inverse_mapping.equal_range(lost_connection); elems.first != elems.second; ++elems.first) + mapping[elems.first->second] = newly_mapped; + } + { + const net::dandelionpp::connection_map cloned = mapper.clone(); + EXPECT_EQ(3u, cloned.size()); + ASSERT_EQ(mapper.end() - mapper.begin(), cloned.end() - cloned.begin()); + for (auto elem : boost::combine(mapper, cloned)) + EXPECT_EQ(boost::get<0>(elem), boost::get<1>(elem)); + } + // mappings should remain evenly distributed amongst 2, with 3 sitting in waiting + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + { + std::map<boost::uuids::uuid, std::size_t> used; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid& out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + EXPECT_EQ(mapping[connection], out); + used[out]++; + } + + EXPECT_EQ(3u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(3u, entry.second); + } + { + const net::dandelionpp::connection_map cloned = mapper.clone(); + EXPECT_EQ(3u, cloned.size()); + ASSERT_EQ(mapper.end() - mapper.begin(), cloned.end() - cloned.begin()); + for (auto elem : boost::combine(mapper, cloned)) + EXPECT_EQ(boost::get<0>(elem), boost::get<1>(elem)); + } +} + +TEST(dandelionpp_map, dropped_connection_remapped) +{ + boost::uuids::random_generator random_uuid{}; + + std::vector<boost::uuids::uuid> connections{3}; + std::generate(connections.begin(), connections.end(), random_uuid); + std::sort(connections.begin(), connections.end()); + + // select 3 of 3 outgoing connections + net::dandelionpp::connection_map mapper{connections, 3}; + EXPECT_EQ(3u, mapper.size()); + EXPECT_EQ(3, mapper.end() - mapper.begin()); + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + EXPECT_FALSE(mapper.update(connections)); + EXPECT_EQ(3u, mapper.size()); + ASSERT_EQ(3, mapper.end() - mapper.begin()); + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + std::map<boost::uuids::uuid, boost::uuids::uuid> mapping; + std::vector<boost::uuids::uuid> in_connections{9}; + std::generate(in_connections.begin(), in_connections.end(), random_uuid); + { + std::map<boost::uuids::uuid, std::size_t> used; + std::multimap<boost::uuids::uuid, boost::uuids::uuid> inverse_mapping; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + EXPECT_TRUE(mapping.emplace(connection, out).second); + inverse_mapping.emplace(out, connection); + used[out]++; + } + + EXPECT_EQ(3u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(3u, entry.second); + + for (const boost::uuids::uuid& connection : in_connections) + EXPECT_EQ(mapping[connection], mapper.get_stem(connection)); + + // drop 1 connection leaving "hole" + const boost::uuids::uuid lost_connection = *(++mapper.begin()); + const auto elem = std::lower_bound(connections.begin(), connections.end(), lost_connection); + ASSERT_NE(connections.end(), elem); + ASSERT_EQ(lost_connection, *elem); + connections.erase(elem); + + EXPECT_TRUE(mapper.update(connections)); + EXPECT_EQ(2u, mapper.size()); + EXPECT_EQ(3, mapper.end() - mapper.begin()); + + for (auto elems = inverse_mapping.equal_range(lost_connection); elems.first != elems.second; ++elems.first) + mapping[elems.first->second] = boost::uuids::nil_uuid(); + } + // remap 3 connections and map 1 new connection to 2 remaining out connections + in_connections.resize(10); + in_connections[9] = random_uuid(); + { + std::map<boost::uuids::uuid, std::size_t> used; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid& out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + used[out]++; + + boost::uuids::uuid& expected = mapping[connection]; + if (!expected.is_nil()) + EXPECT_EQ(expected, out); + else + expected = out; + } + + EXPECT_EQ(2u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(5u, entry.second); + } + // select 3 of 3 connections but do not remap existing links + connections.resize(3); + connections[2] = random_uuid(); + EXPECT_TRUE(mapper.update(connections)); + EXPECT_EQ(3u, mapper.size()); + EXPECT_EQ(3, mapper.end() - mapper.begin()); + { + std::map<boost::uuids::uuid, std::size_t> used; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid& out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + used[out]++; + + EXPECT_EQ(mapping[connection], out); + } + + EXPECT_EQ(2u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(5u, entry.second); + } + // map 8 new incoming connections across 3 outgoing links + in_connections.resize(18); + std::generate(in_connections.begin() + 10, in_connections.end(), random_uuid); + { + std::map<boost::uuids::uuid, std::size_t> used; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid& out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + used[out]++; + + boost::uuids::uuid& expected = mapping[connection]; + if (!expected.is_nil()) + EXPECT_EQ(expected, out); + else + expected = out; + } + + EXPECT_EQ(3u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(6u, entry.second); + } +} + +TEST(dandelionpp_map, dropped_all_connections) +{ + boost::uuids::random_generator random_uuid{}; + + std::vector<boost::uuids::uuid> connections{8}; + std::generate(connections.begin(), connections.end(), random_uuid); + std::sort(connections.begin(), connections.end()); + + // select 3 of 8 outgoing connections + net::dandelionpp::connection_map mapper{connections, 3}; + EXPECT_EQ(3u, mapper.size()); + EXPECT_EQ(3, mapper.end() - mapper.begin()); + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + EXPECT_FALSE(mapper.update(connections)); + EXPECT_EQ(3u, mapper.size()); + ASSERT_EQ(3, mapper.end() - mapper.begin()); + { + std::set<boost::uuids::uuid> used; + for (const boost::uuids::uuid& connection : mapper) + { + EXPECT_FALSE(connection.is_nil()); + EXPECT_TRUE(used.insert(connection).second); + EXPECT_TRUE(std::binary_search(connections.begin(), connections.end(), connection)); + } + } + std::vector<boost::uuids::uuid> in_connections{9}; + std::generate(in_connections.begin(), in_connections.end(), random_uuid); + { + std::map<boost::uuids::uuid, std::size_t> used; + std::map<boost::uuids::uuid, boost::uuids::uuid> mapping; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + EXPECT_TRUE(mapping.emplace(connection, out).second); + used[out]++; + } + + EXPECT_EQ(3u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(3u, entry.second); + + for (const boost::uuids::uuid& connection : in_connections) + EXPECT_EQ(mapping[connection], mapper.get_stem(connection)); + + // drop all connections + connections.clear(); + + EXPECT_TRUE(mapper.update(connections)); + EXPECT_EQ(0u, mapper.size()); + EXPECT_EQ(3, mapper.end() - mapper.begin()); + } + // remap 7 connections to nothing + for (const boost::uuids::uuid& connection : boost::adaptors::slice(in_connections, 0, 7)) + EXPECT_TRUE(mapper.get_stem(connection).is_nil()); + + // select 3 of 30 connections, only 7 should be remapped to new indexes (but all to new uuids) + connections.resize(30); + std::generate(connections.begin(), connections.end(), random_uuid); + EXPECT_TRUE(mapper.update(connections)); + { + std::map<boost::uuids::uuid, std::size_t> used; + for (const boost::uuids::uuid& connection : in_connections) + { + const boost::uuids::uuid& out = mapper.get_stem(connection); + EXPECT_FALSE(out.is_nil()); + used[out]++; + } + + EXPECT_EQ(3u, used.size()); + for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + EXPECT_EQ(3u, entry.second); + } +} diff --git a/tests/unit_tests/output_distribution.cpp b/tests/unit_tests/output_distribution.cpp index 38f442c59..eec694d1e 100644 --- a/tests/unit_tests/output_distribution.cpp +++ b/tests/unit_tests/output_distribution.cpp @@ -93,7 +93,7 @@ bool get_output_distribution(uint64_t amount, uint64_t from, uint64_t to, uint64 crypto::hash get_block_hash(uint64_t height) { - crypto::hash hash; + crypto::hash hash = crypto::null_hash; *((uint64_t*)&hash) = height; return hash; } diff --git a/tests/unit_tests/ringct.cpp b/tests/unit_tests/ringct.cpp index d2b2c3109..075aeac40 100644 --- a/tests/unit_tests/ringct.cpp +++ b/tests/unit_tests/ringct.cpp @@ -788,7 +788,20 @@ TEST(ringct, HPow2) { key G = scalarmultBase(d2h(1)); - key H = hashToPointSimple(G); + // Note that H is computed differently than standard hashing + // This method is not guaranteed to return a curvepoint for all inputs + // Don't use it elsewhere + key H = cn_fast_hash(G); + ge_p3 H_p3; + int decode = ge_frombytes_vartime(&H_p3, H.bytes); + ASSERT_EQ(decode, 0); // this is known to pass for the particular value G + ge_p2 H_p2; + ge_p3_to_p2(&H_p2, &H_p3); + ge_p1p1 H8_p1p1; + ge_mul8(&H8_p1p1, &H_p2); + ge_p1p1_to_p3(&H_p3, &H8_p1p1); + ge_p3_tobytes(H.bytes, &H_p3); + for (int j = 0 ; j < ATOMS ; j++) { ASSERT_TRUE(equalKeys(H, H2[j])); addKeys(H, H, H); diff --git a/utils/build_scripts/android32.Dockerfile b/utils/build_scripts/android32.Dockerfile index e49bdc652..4e4f40276 100644 --- a/utils/build_scripts/android32.Dockerfile +++ b/utils/build_scripts/android32.Dockerfile @@ -1,6 +1,6 @@ FROM debian:stable -RUN apt-get update && apt-get install -y unzip automake build-essential curl file pkg-config git python libtool +RUN apt-get update && apt-get install -y unzip automake build-essential curl file pkg-config git python libtool libtinfo5 WORKDIR /opt/android ## INSTALL ANDROID SDK diff --git a/utils/build_scripts/android64.Dockerfile b/utils/build_scripts/android64.Dockerfile index c4464fa84..90eeb4bae 100644 --- a/utils/build_scripts/android64.Dockerfile +++ b/utils/build_scripts/android64.Dockerfile @@ -1,6 +1,6 @@ FROM debian:stable -RUN apt-get update && apt-get install -y unzip automake build-essential curl file pkg-config git python libtool +RUN apt-get update && apt-get install -y unzip automake build-essential curl file pkg-config git python libtool libtinfo5 WORKDIR /opt/android ## INSTALL ANDROID SDK diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index 36ff3644f..3bbb8b151 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -140,13 +140,14 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(create_wallet) - def get_balance(self, account_index = 0, address_indices = [], all_accounts = False): + def get_balance(self, account_index = 0, address_indices = [], all_accounts = False, strict = False): get_balance = { 'method': 'get_balance', 'params': { 'account_index': account_index, 'address_indices': address_indices, 'all_accounts': all_accounts, + 'strict': strict, }, 'jsonrpc': '2.0', 'id': '0' |