diff options
127 files changed, 7214 insertions, 1112 deletions
diff --git a/ANONYMITY_NETWORKS.md b/ANONYMITY_NETWORKS.md index a5f18010e..feb8528da 100644 --- a/ANONYMITY_NETWORKS.md +++ b/ANONYMITY_NETWORKS.md @@ -19,6 +19,11 @@ network. The transaction will not be broadcast unless an anonymity connection is made or until `monerod` is shutdown and restarted with only public connections enabled. +Anonymity networks can also be used with `monero-wallet-cli` and +`monero-wallet-rpc` - the wallets will connect to a daemon through a proxy. The +daemon must provide a hidden service for the RPC itself, which is separate from +the hidden service for P2P connections. + ## P2P Commands @@ -74,6 +79,35 @@ forwarded to `monerod` localhost port 30000. These addresses will be shared with outgoing peers, over the same network type, otherwise the peer will not be notified of the peer address by the proxy. +### Wallet RPC + +An anonymity network can be configured to forward incoming connections to a +`monerod` RPC port - which is independent from the configuration for incoming +P2P anonymity connections. The anonymity network (Tor/i2p) is +[configured in the same manner](#configuration), except the localhost port +must be the RPC port (typically 18081 for mainnet) instead of the p2p port: + +> HiddenServiceDir /var/lib/tor/data/monero +> HiddenServicePort 18081 127.0.0.1:18081 + +Then the wallet will be configured to use a Tor/i2p address: +> `--proxy 127.0.0.1:9050` +> `--daemon-address rveahdfho7wo4b2m.onion` + +The proxy must match the address type - a Tor proxy will not work properly with +i2p addresses, etc. + +i2p and onion addresses provide the information necessary to authenticate and +encrypt the connection from end-to-end. If desired, SSL can also be applied to +the connection with `--daemon-address https://rveahdfho7wo4b2m.onion` which +requires a server certificate that is signed by a "root" certificate on the +machine running the wallet. Alternatively, `--daemon-cert-file` can be used to +specify a certificate to authenticate the server. + +Proxies can also be used to connect to "clearnet" (ipv4 addresses or ICANN +domains), but `--daemon-cert-file` _must_ be used for authentication and +encryption. + ### Network Types #### Tor & I2P diff --git a/CMakeLists.txt b/CMakeLists.txt index b2d47706b..543e58eca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -887,10 +887,16 @@ find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_ set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES}) if(NOT Boost_FOUND) - die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (1.58) or the equivalent") + die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (>=1.58) or the equivalent") elseif(Boost_FOUND) message(STATUS "Found Boost Version: ${Boost_VERSION}") - if (Boost_VERSION VERSION_LESS 106200 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + if (Boost_VERSION VERSION_LESS 10 AND Boost_VERSION VERSION_LESS 1.62.0 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + set(BOOST_BEFORE_1_62 true) + endif() + if (NOT Boost_VERSION VERSION_LESS 10 AND Boost_VERSION VERSION_LESS 106200 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + set(BOOST_BEFORE_1_62 true) + endif() + if (BOOST_BEFORE_1_62) message(FATAL_ERROR "Boost older than 1.62 is too old to link with OpenSSL 1.1 or newer. " "Update Boost or install OpenSSL 1.0 and set path to it when running cmake: " "cmake -DOPENSSL_ROOT_DIR='/usr/include/openssl-1.0;/usr/lib/openssl-1.0'") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 883d366aa..7b184c00a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,26 +88,26 @@ C4 is meant to provide a reusable optimal collaboration model for open source so ### Preliminaries -- The project SHALL use the git distributed revision control system. -- The project SHALL be hosted on github.com or equivalent, herein called the "Platform". -- The project SHALL use the Platform issue tracker. +- The project MUST use the git distributed revision control system. +- The project MUST be hosted on github.com or equivalent, herein called the "Platform". +- The project MUST use the Platform issue tracker. - Non-GitHub example: - "Platform" could be a vanilla git repo and Trac hosted on the same machine/network. - The Platform issue tracker would be Trac. - The project SHOULD have clearly documented guidelines for code style. - A "Contributor" is a person who wishes to provide a patch, being a set of commits that solve some clearly identified problem. - A "Maintainer" is a person who merges patches to the project. Maintainers are not developers; their job is to enforce process. -- Contributors SHALL NOT have commit access to the repository unless they are also Maintainers. -- Maintainers SHALL have commit access to the repository. -- Everyone, without distinction or discrimination, SHALL have an equal right to become a Contributor under the terms of this contract. +- Contributors MUST NOT have commit access to the repository unless they are also Maintainers. +- Maintainers MUST have commit access to the repository. +- Everyone, without distinction or discrimination, MUST have an equal right to become a Contributor under the terms of this contract. ### Licensing and ownership -- The project SHALL use a share-alike license, such as BSD-3, the GPLv3 or a variant thereof (LGPL, AGPL), or the MPLv2. -- All contributions to the project source code ("patches") SHALL use the same license as the project. -- All patches are owned by their authors. There SHALL NOT be any copyright assignment process. -- The copyrights in the project SHALL be owned collectively by all its Contributors. -- Each Contributor SHALL be responsible for identifying themselves in the project Contributor list. +- The project MUST use a share-alike license, such as BSD-3, the GPLv3 or a variant thereof (LGPL, AGPL), or the MPLv2. +- All contributions to the project source code ("patches") MUST use the same license as the project. +- All patches are owned by their authors. There MUST NOT be any copyright assignment process. +- The copyrights in the project MUST be owned collectively by all its Contributors. +- Each Contributor MUST be responsible for identifying themselves in the project Contributor list. ### Patch requirements @@ -116,27 +116,27 @@ C4 is meant to provide a reusable optimal collaboration model for open source so - A patch SHOULD be a minimal and accurate answer to exactly one identified and agreed problem. - A patch MUST adhere to the code style guidelines of the project if these are defined. - A patch MUST adhere to the "Evolution of Public Contracts" guidelines defined below. -- A patch SHALL NOT include non-trivial code from other projects unless the Contributor is the original author of that code. +- A patch MUST NOT include non-trivial code from other projects unless the Contributor is the original author of that code. - A patch MUST compile cleanly and pass project self-tests on at least the principle target platform. - A patch commit message SHOULD consist of a single short (less than 50 character) line summarizing the change, optionally followed by a blank line and then a more thorough description. - A "Correct Patch" is one that satisfies the above requirements. ### Development process -- Change on the project SHALL be governed by the pattern of accurately identifying problems and applying minimal, accurate solutions to these problems. +- Change on the project MUST be governed by the pattern of accurately identifying problems and applying minimal, accurate solutions to these problems. - To request changes, a user SHOULD log an issue on the project Platform issue tracker. - The user or Contributor SHOULD write the issue by describing the problem they face or observe. - The user or Contributor SHOULD seek consensus on the accuracy of their observation, and the value of solving the problem. -- Users SHALL NOT log feature requests, ideas, or suggestions unrelated to Monero code or Monero's dependency code or Monero's potential/future dependency code or research which successfully implements Monero. -- Users SHALL NOT log any solutions to problems (verifiable or hypothetical) of which are not explicitly documented and/or not provable and/or cannot be reasonably proven. -- Thus, the release history of the project SHALL be a list of meaningful issues logged and solved. -- To work on an issue, a Contributor SHALL fork the project repository and then work on their forked repository. -- To submit a patch, a Contributor SHALL create a Platform pull request back to the project. -- A Contributor SHALL NOT commit changes directly to the project. +- Users MUST NOT log feature requests, ideas, or suggestions unrelated to Monero code or Monero's dependency code or Monero's potential/future dependency code or research which successfully implements Monero. +- Users MUST NOT log any solutions to problems (verifiable or hypothetical) of which are not explicitly documented and/or not provable and/or cannot be reasonably proven. +- Thus, the release history of the project MUST be a list of meaningful issues logged and solved. +- To work on an issue, a Contributor MUST fork the project repository and then work on their forked repository. +- To submit a patch, a Contributor MUST create a Platform pull request back to the project. +- A Contributor MUST NOT commit changes directly to the project. - To discuss a patch, people MAY comment on the Platform pull request, on the commit, or elsewhere. -- To accept or reject a patch, a Maintainer SHALL use the Platform interface. +- To accept or reject a patch, a Maintainer MUST use the Platform interface. - Maintainers SHOULD NOT merge their own patches except in exceptional cases, such as non-responsiveness from other Maintainers for an extended period (more than 30 days) or unless urgent as defined by the Monero Maintainers Team. -- Maintainers SHALL NOT make value judgments on correct patches unless the Maintainer (as may happen in rare circumstances) is a core code developer. +- Maintainers MUST NOT make value judgments on correct patches unless the Maintainer (as may happen in rare circumstances) is a core code developer. - Maintainers MUST NOT merge pull requests in less than 168 hours (1 week) unless deemed urgent by at least 2 people from the Monero Maintainer Team. - The Contributor MAY tag an issue as "Ready" after making a pull request for the issue. - The user who created an issue SHOULD close the issue after checking the patch is successful. @@ -146,27 +146,27 @@ C4 is meant to provide a reusable optimal collaboration model for open source so ### Creating stable releases -- The project SHALL have one branch ("master") that always holds the latest in-progress version and SHOULD always build. -- The project SHALL NOT use topic branches for any reason. Personal forks MAY use topic branches. -- To make a stable release someone SHALL fork the repository by copying it and thus become maintainer of this repository. +- The project MUST have one branch ("master") that always holds the latest in-progress version and SHOULD always build. +- The project MUST NOT use topic branches for any reason. Personal forks MAY use topic branches. +- To make a stable release someone MUST fork the repository by copying it and thus become maintainer of this repository. - Forking a project for stabilization MAY be done unilaterally and without agreement of project maintainers. -- A patch to a stabilization project declared "stable" SHALL be accompanied by a reproducible test case. +- A patch to a stabilization project declared "stable" MUST be accompanied by a reproducible test case. ### Evolution of public contracts -- All Public Contracts (APIs or protocols) SHALL be documented. +- All Public Contracts (APIs or protocols) MUST be documented. - All Public Contracts SHOULD have space for extensibility and experimentation. - A patch that modifies a stable Public Contract SHOULD not break existing applications unless there is overriding consensus on the value of doing this. - A patch that introduces new features to a Public Contract SHOULD do so using new names. - Old names SHOULD be deprecated in a systematic fashion by marking new names as "experimental" until they are stable, then marking the old names as "deprecated". - When sufficient time has passed, old deprecated names SHOULD be marked "legacy" and eventually removed. -- Old names SHALL NOT be reused by new features. +- Old names MUST NOT be reused by new features. - When old names are removed, their implementations MUST provoke an exception (assertion) if used by applications. ### Project administration -- The project founders SHALL act as Administrators to manage the set of project Maintainers. -- The Administrators SHALL ensure their own succession over time by promoting the most effective Maintainers. -- A new Contributor who makes a correct patch SHALL be invited to become a Maintainer. +- The project founders MUST act as Administrators to manage the set of project Maintainers. +- The Administrators MUST ensure their own succession over time by promoting the most effective Maintainers. +- A new Contributor who makes a correct patch MUST be invited to become a Maintainer. - Administrators MAY remove Maintainers who are inactive for an extended period of time, or who repeatedly fail to apply this process accurately. - Administrators SHOULD block or ban "bad actors" who cause stress and pain to others in the project. This should be done after public discussion, with a chance for all parties to speak. A bad actor is someone who repeatedly ignores the rules and culture of the project, who is needlessly argumentative or hostile, or who is offensive, and who is unable to self-correct their behavior when asked to do so by others. diff --git a/Dockerfile b/Dockerfile index 9fe7cfb8f..999290c95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,10 +25,13 @@ RUN set -ex && \ WORKDIR /usr/local +ENV CFLAGS='-fPIC' +ENV CXXFLAGS='-fPIC' + #Cmake -ARG CMAKE_VERSION=3.13.0 -ARG CMAKE_VERSION_DOT=v3.13 -ARG CMAKE_HASH=4058b2f1a53c026564e8936698d56c3b352d90df067b195cb749a97a3d273c90 +ARG CMAKE_VERSION=3.14.0 +ARG CMAKE_VERSION_DOT=v3.14 +ARG CMAKE_HASH=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 RUN set -ex \ && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \ && echo "${CMAKE_HASH} cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \ @@ -39,41 +42,41 @@ RUN set -ex \ && make install ## Boost -ARG BOOST_VERSION=1_68_0 -ARG BOOST_VERSION_DOT=1.68.0 -ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7 +ARG BOOST_VERSION=1_69_0 +ARG BOOST_VERSION_DOT=1.69.0 +ARG BOOST_HASH=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 RUN set -ex \ && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && cd boost_${BOOST_VERSION} \ && ./bootstrap.sh \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="-fPIC" cxxflags="-fPIC" stage + && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="$CFLAGS" cxxflags="$CXXFLAGS" stage ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} # OpenSSL -ARG OPENSSL_VERSION=1.1.0j -ARG OPENSSL_HASH=31bec6c203ce1a8e93d5994f4ed304c63ccf07676118b6634edded12ad1b3246 +ARG OPENSSL_VERSION=1.1.1b +ARG OPENSSL_HASH=5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b RUN set -ex \ && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ && cd openssl-${OPENSSL_VERSION} \ - && ./Configure linux-x86_64 no-shared --static -fPIC \ + && ./Configure linux-x86_64 no-shared --static "$CFLAGS" \ && make build_generated \ && make libcrypto.a \ && make install ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION} # ZMQ -ARG ZMQ_VERSION=v4.2.5 -ARG ZMQ_HASH=d062edd8c142384792955796329baf1e5a3377cd +ARG ZMQ_VERSION=v4.3.1 +ARG ZMQ_HASH=2cb1240db64ce1ea299e00474c646a2453a8435b RUN set -ex \ && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ && cd libzmq \ && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install \ && ldconfig @@ -88,39 +91,39 @@ RUN set -ex \ && mv *.hpp /usr/local/include # Readline -ARG READLINE_VERSION=7.0 -ARG READLINE_HASH=750d437185286f40a369e1e4f4764eda932b9459b5ec9a731628393dd3d32334 +ARG READLINE_VERSION=8.0 +ARG READLINE_HASH=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 RUN set -ex \ && curl -s -O https://ftp.gnu.org/gnu/readline/readline-${READLINE_VERSION}.tar.gz \ && echo "${READLINE_HASH} readline-${READLINE_VERSION}.tar.gz" | sha256sum -c \ && tar -xzf readline-${READLINE_VERSION}.tar.gz \ && cd readline-${READLINE_VERSION} \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \ + && ./configure \ && make \ && make install # Sodium -ARG SODIUM_VERSION=1.0.16 -ARG SODIUM_HASH=675149b9b8b66ff44152553fb3ebf9858128363d +ARG SODIUM_VERSION=1.0.17 +ARG SODIUM_HASH=b732443c442239c2e0184820e9b23cca0de0828c RUN set -ex \ && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ && cd libsodium \ && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \ + && ./configure \ && make \ && make check \ && make install # Udev -ARG UDEV_VERSION=v3.2.6 -ARG UDEV_HASH=0c35b136c08d64064efa55087c54364608e65ed6 +ARG UDEV_VERSION=v3.2.7 +ARG UDEV_HASH=4758e346a14126fc3a964de5831e411c27ebe487 RUN set -ex \ && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ && cd eudev \ && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ + && ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ && make \ && make install @@ -132,7 +135,7 @@ RUN set -ex \ && cd libusb \ && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared \ + && ./configure --disable-shared \ && make \ && make install @@ -144,20 +147,20 @@ RUN set -ex \ && cd hidapi \ && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ && ./bootstrap \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install # Protobuf -ARG PROTOBUF_VERSION=v3.6.1 -ARG PROTOBUF_HASH=48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 +ARG PROTOBUF_VERSION=v3.7.0 +ARG PROTOBUF_HASH=582743bf40c5d3639a70f98f183914a2c0cd0680 RUN set -ex \ && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ && cd protobuf \ && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \ && git submodule update --init --recursive \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install \ && ldconfig @@ -185,8 +188,14 @@ RUN set -ex && \ rm -rf /var/lib/apt COPY --from=builder /src/build/release/bin /usr/local/bin/ +# Create monero user +RUN adduser --system --group --disabled-password monero && \ + mkdir -p /wallet /home/monero/.bitmonero && \ + chown -R monero:monero /home/monero/.bitmonero && \ + chown -R monero:monero /wallet + # Contains the blockchain -VOLUME /root/.bitmonero +VOLUME /home/monero/.bitmonero # Generate your wallet via accessing the container and run: # cd /wallet @@ -196,5 +205,8 @@ VOLUME /wallet EXPOSE 18080 EXPOSE 18081 +# switch to user monero +USER monero + ENTRYPOINT ["monerod", "--p2p-bind-ip=0.0.0.0", "--p2p-bind-port=18080", "--rpc-bind-ip=0.0.0.0", "--rpc-bind-port=18081", "--non-interactive", "--confirm-external-bind"] @@ -29,3 +29,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Parts of the project are originally copyright (c) 2012-2013 The Cryptonote developers + +Parts of the project are originally copyright (c) 2014 The Boolberry +developers, distributed under the MIT licence: + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + @@ -73,6 +73,8 @@ Monero is a private, secure, untraceable, decentralised digital currency. You ar **Untraceability:** By taking advantage of ring signatures, a special property of a certain type of cryptography, Monero is able to ensure that transactions are not only untraceable, but have an optional measure of ambiguity that ensures that transactions cannot easily be tied back to an individual user or computer. +**Decentralization:** The utility of monero depends on its decentralised peer-to-peer consensus network - anyone should be able to run the monero software, validate the integrity of the blockchain, and participate in all aspects of the monero network using consumer-grade commodity hardware. Decentralization of the monero network is maintained by software development that minimizes the costs of running the monero software and inhibits the proliferation of specialized, non-commodity hardware. + ## About this project This is the core implementation of Monero. It is open source and completely free to use without restrictions, except for those specified in the license agreement below. There are no restrictions on anyone creating an alternative implementation of Monero that uses the protocol and network in a compatible manner. diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 6e7efd1d7..e07e16d91 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -357,6 +357,7 @@ eof: if (m_stdin_reader.eos()) { MGINFO("EOF on stdin, exiting"); + std::cout << std::endl; break; } if (!get_line_ret) diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index f5548c585..67c63cca5 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -58,6 +58,7 @@ #define DEFAULT_TIMEOUT_MS_REMOTE 300000 // 5 minutes #define TIMEOUT_EXTRA_MS_PER_BYTE 0.2 + PRAGMA_WARNING_PUSH namespace epee { @@ -99,7 +100,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_connection_type( connection_type ), m_throttle_speed_in("speed_in", "throttle_speed_in"), m_throttle_speed_out("speed_out", "throttle_speed_out"), - m_timer(socket_.get_io_service()), + m_timer(GET_IO_SERVICE(socket_)), m_local(false), m_ready_to_close(false) { @@ -243,7 +244,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> boost::asio::io_service& connection<t_protocol_handler>::get_io_service() { - return socket().get_io_service(); + return GET_IO_SERVICE(socket()); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -487,7 +488,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(!m_is_multithreaded) { //single thread model, we can wait in blocked call - size_t cnt = socket().get_io_service().run_one(); + size_t cnt = GET_IO_SERVICE(socket()).run_one(); if(!cnt)//service is going to quit return false; }else @@ -497,7 +498,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //if no handlers were called //TODO: Maybe we need to have have critical section + event + callback to upper protocol to //ask it inside(!) critical region if we still able to go in event wait... - size_t cnt = socket().get_io_service().poll_one(); + size_t cnt = GET_IO_SERVICE(socket()).poll_one(); if(!cnt) misc_utils::sleep_no_w(1); } @@ -670,9 +671,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size"); reset_timer(get_default_timeout(), false); async_write(boost::asio::buffer(m_send_que.front().data(), size_now ) , - //strand_.wrap( + strand_.wrap( boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) - //) + ) ); //_dbg3("(chunk): " << size_now); //logger_handle_net_write(size_now); @@ -761,8 +762,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) // Initiate graceful connection closure. m_timer.cancel(); boost::system::error_code ignored_ec; + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + socket_.shutdown(ignored_ec); socket().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); - socket().close(); if (!m_host.empty()) { try { host_count(m_host, -1); } catch (...) { /* ignore */ } @@ -855,9 +857,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H))) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), void(), "Unexpected queue size"); async_write(boost::asio::buffer(m_send_que.front().data(), size_now) , - // strand_.wrap( + strand_.wrap( boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2) - // ) + ) ); //_dbg3("(normal)" << size_now); } @@ -1207,7 +1209,7 @@ POP_WARNINGS template<class t_protocol_handler> bool boosted_tcp_server<t_protocol_handler>::add_connection(t_connection_context& out, boost::asio::ip::tcp::socket&& sock, network_address real_remote, epee::net_utils::ssl_support_t ssl_support) { - if(std::addressof(get_io_service()) == std::addressof(sock.get_io_service())) + if(std::addressof(get_io_service()) == std::addressof(GET_IO_SERVICE(sock))) { connection_ptr conn(new connection<t_protocol_handler>(std::move(sock), m_state, m_connection_type, ssl_support, m_ssl_context)); if(conn->start(false, 1 < m_threads_count, std::move(real_remote))) diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 58a8e6888..f0425278d 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -327,10 +327,17 @@ namespace net_utils m_net_client.set_ssl(m_ssl_support, m_ssl_private_key_and_certificate_path, m_ssl_allowed_certificates, m_ssl_allowed_fingerprints, m_ssl_allow_any_cert); } + template<typename F> + void set_connector(F connector) + { + CRITICAL_REGION_LOCAL(m_lock); + m_net_client.set_connector(std::move(connector)); + } + bool connect(std::chrono::milliseconds timeout) { CRITICAL_REGION_LOCAL(m_lock); - return m_net_client.connect(m_host_buff, m_port, timeout, "0.0.0.0"); + return m_net_client.connect(m_host_buff, m_port, timeout); } //--------------------------------------------------------------------------- bool disconnect() diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 742cf916e..aa3df7160 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -33,12 +33,17 @@ //#include <Ws2tcpip.h> #include <string> #include <boost/version.hpp> -#include <boost/asio.hpp> +#include <boost/asio/io_service.hpp> +#include <boost/asio/ip/tcp.hpp> +#include <boost/asio/read.hpp> #include <boost/asio/ssl.hpp> #include <boost/asio/steady_timer.hpp> +#include <boost/thread/future.hpp> #include <boost/lambda/bind.hpp> #include <boost/lambda/lambda.hpp> #include <boost/interprocess/detail/atomic.hpp> +#include <boost/system/error_code.hpp> +#include <functional> #include "net/net_utils_base.h" #include "net/net_ssl.h" #include "misc_language.h" @@ -55,6 +60,12 @@ namespace epee { namespace net_utils { + struct direct_connect + { + boost::unique_future<boost::asio::ip::tcp::socket> + operator()(const std::string& addr, const std::string& port, boost::asio::steady_timer&) const; + }; + class blocked_mode_client { @@ -85,31 +96,38 @@ namespace net_utils ref_bytes_transferred = bytes_transferred; } }; - + public: inline - blocked_mode_client():m_initialized(false), - m_connected(false), - m_deadline(m_io_service), - m_shutdowned(0), - m_ssl_support(epee::net_utils::ssl_support_t::e_ssl_support_autodetect), - m_ctx({boost::asio::ssl::context(boost::asio::ssl::context::tlsv12), {}}), - m_ssl_socket(new boost::asio::ssl::stream<boost::asio::ip::tcp::socket>(m_io_service,m_ctx.context)) + blocked_mode_client() : + m_io_service(), + m_ctx({boost::asio::ssl::context(boost::asio::ssl::context::tlsv12), {}}), + m_connector(direct_connect{}), + m_ssl_socket(new boost::asio::ssl::stream<boost::asio::ip::tcp::socket>(m_io_service, m_ctx.context)), + m_ssl_support(epee::net_utils::ssl_support_t::e_ssl_support_autodetect), + m_initialized(true), + m_connected(false), + m_deadline(m_io_service), + m_shutdowned(0) { - - - m_initialized = true; + } + /*! The first/second parameters are host/port respectively. The third + parameter is for setting the timeout callback - the timer is + already set by the caller, the callee only needs to set the + behavior. - // No deadline is required until the first socket operation is started. We - // set the deadline to positive infinity so that the actor takes no action - // until a specific deadline is set. - m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); + Additional asynchronous operations should be queued using the + `io_service` from the timer. The implementation should assume + multi-threaded I/O processing. - // Start the persistent actor that checks for deadline expiry. - check_deadline(); + If the callee cannot start an asynchronous operation, an exception + should be thrown to signal an immediate failure. + + The return value is a future to a connected socket. Asynchronous + failures should use the `set_exception` method. */ + using connect_func = boost::unique_future<boost::asio::ip::tcp::socket>(const std::string&, const std::string&, boost::asio::steady_timer&); - } inline ~blocked_mode_client() { @@ -128,33 +146,28 @@ namespace net_utils } inline - bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout, const std::string& bind_ip = "0.0.0.0") + bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout) { - return connect(addr, std::to_string(port), timeout, bind_ip); + return connect(addr, std::to_string(port), timeout); } inline - try_connect_result_t try_connect(const std::string& addr, const std::string& port, const boost::asio::ip::tcp::endpoint &remote_endpoint, std::chrono::milliseconds timeout, const std::string& bind_ip, epee::net_utils::ssl_support_t ssl_support) + try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, epee::net_utils::ssl_support_t ssl_support) { - m_ssl_socket->next_layer().open(remote_endpoint.protocol()); - if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) - { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(addr.c_str()), 0); - m_ssl_socket->next_layer().bind(local_endpoint); - } - - m_deadline.expires_from_now(timeout); + boost::unique_future<boost::asio::ip::tcp::socket> connection = m_connector(addr, port, m_deadline); + for (;;) + { + m_io_service.reset(); + m_io_service.run_one(); - boost::system::error_code ec = boost::asio::error::would_block; - - m_ssl_socket->next_layer().async_connect(remote_endpoint, boost::lambda::var(ec) = boost::lambda::_1); - while (ec == boost::asio::error::would_block) - { - m_io_service.run_one(); + if (connection.is_ready()) + break; } - - if (!ec && m_ssl_socket->next_layer().is_open()) + + m_ssl_socket->next_layer() = connection.get(); + m_deadline.cancel(); + if (m_ssl_socket->next_layer().is_open()) { m_connected = true; m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); @@ -183,14 +196,14 @@ namespace net_utils return CONNECT_SUCCESS; }else { - MWARNING("Some problems at connect, message: " << ec.message()); + MWARNING("Some problems at connect, expected open socket"); return CONNECT_FAILURE; } } inline - bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, const std::string& bind_ip = "0.0.0.0") + bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) { m_connected = false; try @@ -205,25 +218,7 @@ namespace net_utils // Get a list of endpoints corresponding to the server name. - ////////////////////////////////////////////////////////////////////////// - - boost::asio::ip::tcp::resolver resolver(m_io_service); - boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); - boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) - { - LOG_ERROR("Failed to resolve " << addr); - return false; - } - - ////////////////////////////////////////////////////////////////////////// - - - //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); - boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - - try_connect_result_t try_connect_result = try_connect(addr, port, remote_endpoint, timeout, bind_ip, m_ssl_support); + try_connect_result_t try_connect_result = try_connect(addr, port, timeout, m_ssl_support); if (try_connect_result == CONNECT_FAILURE) return false; if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) @@ -233,7 +228,7 @@ namespace net_utils { MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; - if (try_connect(addr, port, remote_endpoint, timeout, bind_ip, m_ssl_support) != CONNECT_SUCCESS) + if (try_connect(addr, port, timeout, m_ssl_support) != CONNECT_SUCCESS) return false; } } @@ -251,6 +246,11 @@ namespace net_utils return true; } + //! Change the connection routine (proxy, etc.) + void set_connector(std::function<connect_func> connector) + { + m_connector = std::move(connector); + } inline bool disconnect() @@ -265,7 +265,6 @@ namespace net_utils m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both); } } - catch(const boost::system::system_error& /*er*/) { //LOG_ERROR("Some problems at disconnect, message: " << er.what()); @@ -304,6 +303,7 @@ namespace net_utils // Block until the asynchronous operation has completed. while (ec == boost::asio::error::would_block) { + m_io_service.reset(); m_io_service.run_one(); } @@ -433,6 +433,7 @@ namespace net_utils // Block until the asynchronous operation has completed. while (ec == boost::asio::error::would_block && !boost::interprocess::ipcdetail::atomic_read32(&m_shutdowned)) { + m_io_service.reset(); m_io_service.run_one(); } @@ -573,10 +574,6 @@ namespace net_utils return true; } - void set_connected(bool connected) - { - m_connected = connected; - } boost::asio::io_service& get_io_service() { return m_io_service; @@ -619,6 +616,7 @@ namespace net_utils m_ssl_socket->async_shutdown(boost::lambda::var(ec) = boost::lambda::_1); while (ec == boost::asio::error::would_block) { + m_io_service.reset(); m_io_service.run_one(); } // Ignore "short read" error @@ -665,11 +663,8 @@ namespace net_utils boost::asio::io_service m_io_service; epee::net_utils::ssl_context_t m_ctx; std::shared_ptr<boost::asio::ssl::stream<boost::asio::ip::tcp::socket>> m_ssl_socket; + std::function<connect_func> m_connector; epee::net_utils::ssl_support_t m_ssl_support; - std::string m_ssl_private_key; - std::string m_ssl_certificate; - std::list<std::string> m_ssl_allowed_certificates; - bool m_ssl_allow_any_cerl; bool m_initialized; bool m_connected; boost::asio::steady_timer m_deadline; @@ -790,3 +785,4 @@ namespace net_utils }; } } + diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 7b5b07ef2..50536f63b 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -44,6 +44,12 @@ #define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) #endif +#if BOOST_VERSION >= 107000 +#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context()) +#else +#define GET_IO_SERVICE(s) ((s).get_io_service()) +#endif + namespace net { class tor_address; diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index da84fd8ea..ca7c81ddc 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -82,6 +82,7 @@ namespace epee struct array_entry_t { array_entry_t():m_it(m_array.end()){} + array_entry_t(const array_entry_t& other):m_array(other.m_array), m_it(m_array.end()){} const t_entry_type* get_first_val() const { diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 0787a9d08..2465afebb 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,8 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c +add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp) + if (USE_READLINE AND GNU_READLINE_FOUND) add_library(epee_readline STATIC readline_buffer.cpp) endif() diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index 83db171d8..cafc08ead 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -47,6 +47,12 @@ // TODO: #include "net/network_throttle-detail.hpp" +#if BOOST_VERSION >= 107000 +#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context()) +#else +#define GET_IO_SERVICE(s) ((s).get_io_service()) +#endif + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.conn" @@ -117,8 +123,8 @@ connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::s : m_stats(std::move(stats)), mI( new connection_basic_pimpl("peer") ), - strand_(sock.get_io_service()), - socket_(sock.get_io_service(), ssl_context.context), + strand_(GET_IO_SERVICE(sock)), + socket_(GET_IO_SERVICE(sock), ssl_context.context), m_want_close_connection(false), m_was_shutdown(false), m_ssl_support(ssl_support), @@ -167,7 +173,6 @@ connection_basic::~connection_basic() noexcept(false) { std::string remote_addr_str = "?"; try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ; _note("Destructing connection #"<<mI->m_peer_number << " to " << remote_addr_str); -try { throw 0; } catch(...){} } void connection_basic::set_rate_up_limit(uint64_t limit) { diff --git a/contrib/epee/src/net_helper.cpp b/contrib/epee/src/net_helper.cpp new file mode 100644 index 000000000..3543f5716 --- /dev/null +++ b/contrib/epee/src/net_helper.cpp @@ -0,0 +1,54 @@ +#include "net/net_helper.h" + +namespace epee +{ +namespace net_utils +{ + boost::unique_future<boost::asio::ip::tcp::socket> + direct_connect::operator()(const std::string& addr, const std::string& port, boost::asio::steady_timer& timeout) const + { + // Get a list of endpoints corresponding to the server name. + ////////////////////////////////////////////////////////////////////////// + boost::asio::ip::tcp::resolver resolver(GET_IO_SERVICE(timeout)); + boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::asio::ip::tcp::resolver::iterator end; + if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty + throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + + ////////////////////////////////////////////////////////////////////////// + + struct new_connection + { + boost::promise<boost::asio::ip::tcp::socket> result_; + boost::asio::ip::tcp::socket socket_; + + explicit new_connection(boost::asio::io_service& io_service) + : result_(), socket_(io_service) + {} + }; + + const auto shared = std::make_shared<new_connection>(GET_IO_SERVICE(timeout)); + timeout.async_wait([shared] (boost::system::error_code error) + { + if (error != boost::system::errc::operation_canceled && shared && shared->socket_.is_open()) + { + shared->socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both); + shared->socket_.close(); + } + }); + shared->socket_.async_connect(*iterator, [shared] (boost::system::error_code error) + { + if (shared) + { + if (error) + shared->result_.set_exception(boost::system::system_error{error}); + else + shared->result_.set_value(std::move(shared->socket_)); + } + }); + return shared->result_.get_future(); + } +} +} + diff --git a/external/trezor-common b/external/trezor-common -Subproject 588f8e03f5ac111adf719f0a437de67481a26ae +Subproject cb238cb1f134accc4200217d9511115a8f61c6c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2ae72ee81..da6d76d97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,6 +109,7 @@ add_subdirectory(ringct) add_subdirectory(checkpoints) add_subdirectory(cryptonote_basic) add_subdirectory(cryptonote_core) +add_subdirectory(lmdb) add_subdirectory(multisig) add_subdirectory(net) if(NOT IOS) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 313e716e0..d2fe39fc2 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -947,6 +947,17 @@ public: virtual size_t get_block_weight(const uint64_t& height) const = 0; /** + * @brief fetch the last N blocks' weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const = 0; + + /** * @brief fetch a block's cumulative difficulty * * The subclass should return the cumulative difficulty of the block with the @@ -1000,6 +1011,17 @@ public: virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; /** + * @brief fetch the last N blocks' long term weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const = 0; + + /** * @brief fetch a block's hash * * The subclass should return hash of the block with the diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index becffed16..9f71fd068 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -54,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 4 +#define VERSION 5 namespace { @@ -274,7 +274,7 @@ typedef struct mdb_block_info_1 uint64_t bi_timestamp; uint64_t bi_coins; uint64_t bi_weight; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_diff; crypto::hash bi_hash; } mdb_block_info_1; @@ -284,7 +284,7 @@ typedef struct mdb_block_info_2 uint64_t bi_timestamp; uint64_t bi_coins; uint64_t bi_weight; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; } mdb_block_info_2; @@ -295,13 +295,26 @@ typedef struct mdb_block_info_3 uint64_t bi_timestamp; uint64_t bi_coins; uint64_t bi_weight; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; uint64_t bi_long_term_block_weight; } mdb_block_info_3; -typedef mdb_block_info_3 mdb_block_info; +typedef struct mdb_block_info_4 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff_lo; + uint64_t bi_diff_hi; + crypto::hash bi_hash; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_4; + +typedef mdb_block_info_4 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; @@ -757,7 +770,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; bi.bi_weight = block_weight; - bi.bi_diff = cumulative_difficulty; + bi.bi_diff_hi = (cumulative_difficulty >> 64).convert_to<uint64_t>(); + bi.bi_diff_lo = (cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; if (blk.major_version >= 4) @@ -2445,6 +2459,70 @@ size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const return ret; } +std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + const uint64_t h = height(); + if (start_height >= h) + throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str())); + + std::vector<uint64_t> ret; + ret.reserve(count); + + MDB_val v; + uint64_t range_begin = 0, range_end = 0; + for (uint64_t height = start_height; height < h && count--; ++height) + { + if (height >= range_begin && height < range_end) + { + // nothing to do + } + else + { + int result = 0; + if (range_end > 0) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str())); + } + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); + ret.push_back(*(const uint64_t*)(((const char*)bi) + offset)); + } + + TXN_POSTFIX_RDONLY(); + return ret; +} + +std::vector<uint64_t> BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight)); +} + +std::vector<uint64_t> BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight)); +} + difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); @@ -2463,7 +2541,9 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - difficulty_type ret = bi->bi_diff; + difficulty_type ret = bi->bi_diff_hi; + ret <<= 64; + ret |= bi->bi_diff_lo; TXN_POSTFIX_RDONLY(); return ret; } @@ -4976,6 +5056,133 @@ void BlockchainLMDB::migrate_3_4() txn.commit(); } +void BlockchainLMDB::migrate_4_5() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 4 to 5 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_3 *bi_old = (const mdb_block_info_3*)v.mv_data; + mdb_block_info_4 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff_lo = bi_old->bi_diff; + bi.bi_diff_hi = 0; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 5; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { if (oldversion < 1) @@ -4986,6 +5193,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_2_3(); if (oldversion < 4) migrate_3_4(); + if (oldversion < 5) + migrate_4_5(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 7a04c5f0c..2f89b77ac 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -219,6 +219,8 @@ public: virtual size_t get_block_weight(const uint64_t& height) const; + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const; + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; virtual difficulty_type get_block_difficulty(const uint64_t& height) const; @@ -227,6 +229,8 @@ public: virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; @@ -394,6 +398,8 @@ private: virtual uint64_t get_database_size() const; + std::vector<uint64_t> get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const; + // fix up anything that may be wrong due to past bugs virtual void fixup(); @@ -412,6 +418,9 @@ private: // migrate from DB version 3 to 4 void migrate_3_4(); + // migrate from DB version 4 to 5 + void migrate_4_5(); + void cleanup_batch(); private: diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index eb9c3e45a..7916364c5 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -72,10 +72,12 @@ public: virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; } virtual uint64_t get_top_block_timestamp() const { return 0; } virtual size_t get_block_weight(const uint64_t& height) const { return 128; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const { return {}; } virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; } virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; } virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; } virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const { return {}; } virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<cryptonote::block>(); } virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<crypto::hash>(); } diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index e4efdc3cb..8454595ac 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -294,7 +294,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path } // 4 byte magic + (currently) 1024 byte header structures - bootstrap.seek_to_first_chunk(import_file); + uint8_t major_version, minor_version; + bootstrap.seek_to_first_chunk(import_file, major_version, minor_version); std::string str1; char buffer1[1024]; @@ -415,7 +416,23 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { str1.assign(buffer_block, chunk_size); bootstrap::block_package bp; - if (! ::serialization::parse_binary(str1, bp)) + bool res; + if (major_version == 0) + { + bootstrap::block_package_1 bp1; + res = ::serialization::parse_binary(str1, bp1); + if (res) + { + bp.block = std::move(bp1.block); + bp.txs = std::move(bp1.txs); + bp.block_weight = bp1.block_weight; + bp.cumulative_difficulty = bp1.cumulative_difficulty; + bp.coins_generated = bp1.coins_generated; + } + } + else + res = ::serialization::parse_binary(str1, bp); + if (!res) throw std::runtime_error("Error in deserialization of chunk"); int display_interval = 1000; diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index fb9a24f5d..252c79776 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -124,8 +124,8 @@ bool BootstrapFile::initialize_file() *m_raw_data_file << blob; bootstrap::file_info bfi; - bfi.major_version = 0; - bfi.minor_version = 1; + bfi.major_version = 1; + bfi.minor_version = 0; bfi.header_size = header_size; bootstrap::blocks_info bbi; @@ -323,7 +323,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem return BootstrapFile::close(); } -uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) +uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version) { uint32_t file_magic; @@ -371,6 +371,8 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) uint64_t full_header_size = sizeof(file_magic) + bfi.header_size; import_file.seekg(full_header_size); + major_version = bfi.major_version; + minor_version = bfi.minor_version; return full_header_size; } @@ -461,7 +463,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s } uint64_t full_header_size; // 4 byte magic + length of header structures - full_header_size = seek_to_first_chunk(import_file); + uint8_t major_version, minor_version; + full_header_size = seek_to_first_chunk(import_file, major_version, minor_version); MINFO("Scanning blockchain from bootstrap file..."); bool quit = false; diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 5fb2cf366..1e6ef5d81 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -60,7 +60,7 @@ public: uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit); uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height); uint64_t count_blocks(const std::string& dir_path); - uint64_t seek_to_first_chunk(std::ifstream& import_file); + uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version); bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, boost::filesystem::path& output_file, uint64_t use_block_height=0); diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 554c6d56e..70b3eea7e 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -29,7 +29,7 @@ #pragma once #include "cryptonote_basic/cryptonote_boost_serialization.h" -#include "cryptonote_basic/difficulty.h" +#include "serialization/difficulty_type.h" namespace cryptonote @@ -66,6 +66,23 @@ namespace cryptonote END_SERIALIZE() }; + struct block_package_1 + { + cryptonote::block block; + std::vector<transaction> txs; + size_t block_weight; + uint64_t cumulative_difficulty; + uint64_t coins_generated; + + BEGIN_SERIALIZE() + FIELD(block) + FIELD(txs) + VARINT_FIELD(block_weight) + VARINT_FIELD(cumulative_difficulty) + VARINT_FIELD(coins_generated) + END_SERIALIZE() + }; + struct block_package { cryptonote::block block; @@ -78,7 +95,7 @@ namespace cryptonote FIELD(block) FIELD(txs) VARINT_FIELD(block_weight) - VARINT_FIELD(cumulative_difficulty) + FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() }; diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 711a8ba30..1a1155c7c 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -32,9 +32,9 @@ #include <stdlib.h> #include "include_base_utils.h" +#include "common/threadpool.h" #include <random> #include <boost/thread/mutex.hpp> -#include <boost/thread/thread.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/optional.hpp> using namespace epee; @@ -293,9 +293,8 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_delete(m_data->m_ub_context); m_data->m_ub_context = ub_ctx_create(); add_anchors(m_data->m_ub_context); - dns_public_addr = tools::dns_utils::parse_dns_public(DNS_PUBLIC); - for (const auto &ip: dns_public_addr) - ub_ctx_set_fwd(m_data->m_ub_context, string_copy(ip.c_str())); + for (const auto &ip: DEFAULT_DNS_PUBLIC_ADDR) + ub_ctx_set_fwd(m_data->m_ub_context, string_copy(ip)); ub_ctx_set_option(m_data->m_ub_context, string_copy("do-udp:"), string_copy("no")); ub_ctx_set_option(m_data->m_ub_context, string_copy("do-tcp:"), string_copy("yes")); } @@ -524,16 +523,16 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std size_t first_index = dis(gen); // send all requests in parallel - std::vector<boost::thread> threads(dns_urls.size()); std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; for (size_t n = 0; n < dns_urls.size(); ++n) { - threads[n] = boost::thread([n, dns_urls, &records, &avail, &valid](){ + tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); }); } - for (size_t n = 0; n < dns_urls.size(); ++n) - threads[n].join(); + waiter.wait(&tpool); size_t cur_index = first_index; do diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 84f002929..d22d59b36 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -39,6 +39,7 @@ set(crypto_sources hash-extra-jh.c hash-extra-skein.c hash.c + hmac-keccak.c jh.c keccak.c oaes_lib.c @@ -64,6 +65,7 @@ set(crypto_private_headers groestl_tables.h hash-ops.h hash.h + hmac-keccak.h initializer.h jh.h keccak.h diff --git a/src/crypto/CryptonightR_JIT.c b/src/crypto/CryptonightR_JIT.c index 68258a959..ee8f3f36f 100644 --- a/src/crypto/CryptonightR_JIT.c +++ b/src/crypto/CryptonightR_JIT.c @@ -4,6 +4,9 @@ #include <string.h> #include <stdio.h> #include <unistd.h> +#if !(defined(_MSC_VER) || defined(__MINGW32__)) +#include <sys/mman.h> +#endif #include "int-util.h" #include "hash-ops.h" @@ -58,6 +61,11 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f uint8_t* JIT_code = (uint8_t*) buf; const uint8_t* JIT_code_end = JIT_code + buf_size; +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_WRITE)) + return 1; +#endif + APPEND_CODE(prologue, sizeof(prologue)); uint32_t prev_rot_src = 0xFFFFFFFFU; @@ -101,6 +109,11 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f APPEND_CODE(epilogue, sizeof(epilogue)); +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_EXEC)) + return 1; +#endif + __builtin___clear_cache((char*)buf, (char*)JIT_code); return 0; diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index efdeef8d1..6d4905ad5 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -33,11 +33,11 @@ extern "C" #define RC_LENGTH (5 * (AES_BLOCK_SIZE / 4 - 2)) #if defined(_MSC_VER) -#define ALIGN __declspec(align(TABLE_ALIGN)) +#define LOCAL_ALIGN __declspec(align(TABLE_ALIGN)) #elif defined(__GNUC__) -#define ALIGN __attribute__ ((aligned(16))) +#define LOCAL_ALIGN __attribute__ ((aligned(16))) #else -#define ALIGN +#define LOCAL_ALIGN #endif #define rf1(r,c) (r) @@ -131,7 +131,7 @@ extern "C" #define t_set(m,n) t_##m##n #define t_use(m,n) t_##m##n -#define d_4(t,n,b,e,f,g,h) ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } +#define d_4(t,n,b,e,f,g,h) LOCAL_ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } #define four_tables(x,tab,vf,rf,c) \ (tab[0][bval(vf(x,0,c),rf(0,c))] \ diff --git a/src/crypto/hmac-keccak.c b/src/crypto/hmac-keccak.c new file mode 100644 index 000000000..edcb2065e --- /dev/null +++ b/src/crypto/hmac-keccak.c @@ -0,0 +1,81 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "hmac-keccak.h" +#include "memwipe.h" + +#define KECCAK_BLOCKLEN 136 +#define HASH_SIZE 32 + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen) { + const uint8_t *key = _key; + uint8_t keyhash[HASH_SIZE]; + uint8_t pad[KECCAK_BLOCKLEN]; + uint64_t i; + + if (keylen > KECCAK_BLOCKLEN) { + keccak(key, keylen, keyhash, HASH_SIZE); + key = keyhash; + keylen = HASH_SIZE; + } + + keccak_init(&S->inner); + memset(pad, 0x36, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->inner, pad, KECCAK_BLOCKLEN); + + keccak_init(&S->outer); + memset(pad, 0x5c, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->outer, pad, KECCAK_BLOCKLEN); + + memwipe(keyhash, HASH_SIZE); +} + +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen) { + keccak_update(&S->inner, data, datalen); +} + +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest) { + uint8_t ihash[HASH_SIZE]; + keccak_finish(&S->inner, ihash); + keccak_update(&S->outer, ihash, HASH_SIZE); + keccak_finish(&S->outer, digest); + memwipe(ihash, HASH_SIZE); +} + +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen) { + hmac_keccak_state S; + hmac_keccak_init(&S, key, keylen); + hmac_keccak_update(&S, in, inlen); + hmac_keccak_finish(&S, out); +} diff --git a/src/crypto/hmac-keccak.h b/src/crypto/hmac-keccak.h new file mode 100644 index 000000000..c450860d4 --- /dev/null +++ b/src/crypto/hmac-keccak.h @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef HMAC_KECCAK_H +#define HMAC_KECCAK_H + +#include "keccak.h" + +// HMAC RFC 2104 with Keccak-256 base hash function +// +// B = KECCAK_BLOCKLEN = 136 B +// L = HASH_SIZE = 32 B +// +// Note this is not HMAC-SHA3 as SHA3 and Keccak differs in +// the padding constant. + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + KECCAK_CTX inner; + KECCAK_CTX outer; +} hmac_keccak_state; + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen); +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen); +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest); +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen); + +#ifdef __cplusplus +} +#endif +#endif //HMAC_KECCAK_H diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 9dc6f5038..7f36c9dc3 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -76,7 +76,7 @@ static inline int use_v4_jit(void) const char *env = getenv("MONERO_USE_CNV4_JIT"); if (!env) { - use_v4_jit_flag = 0; + use_v4_jit_flag = 1; } else if (!strcmp(env, "0") || !strcmp(env, "no")) { use_v4_jit_flag = 0; @@ -274,10 +274,10 @@ static inline int use_v4_jit(void) #define VARIANT2_2() \ do if (variant == 2 || variant == 3) \ { \ - *U64(hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ - *(U64(hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ - hi ^= SWAP64LE(*U64(hp_state + (j ^ 0x20))); \ - lo ^= SWAP64LE(*(U64(hp_state + (j ^ 0x20)) + 1)); \ + *U64(local_hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ + *(U64(local_hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ + hi ^= SWAP64LE(*U64(local_hp_state + (j ^ 0x20))); \ + lo ^= SWAP64LE(*(U64(local_hp_state + (j ^ 0x20)) + 1)); \ } while (0) #define V4_REG_LOAD(dst, src) \ @@ -405,7 +405,7 @@ static inline int use_v4_jit(void) #define pre_aes() \ j = state_index(a); \ - _c = _mm_load_si128(R128(&hp_state[j])); \ + _c = _mm_load_si128(R128(&local_hp_state[j])); \ _a = _mm_load_si128(R128(a)); \ /* @@ -418,20 +418,20 @@ static inline int use_v4_jit(void) * This code is based upon an optimized implementation by dga. */ #define post_aes() \ - VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ - VARIANT1_1(&hp_state[j]); \ + _mm_store_si128(R128(&local_hp_state[j]), _mm_xor_si128(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_INTEGER_MATH_SSE2(b, c); \ VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ - VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ @@ -756,10 +756,10 @@ void slow_hash_allocate_state(void) #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, 0, 0); + MAP_PRIVATE | MAP_ANON, -1, 0); #else hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 0, 0); + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); #endif if(hp_state == MAP_FAILED) hp_state = NULL; @@ -778,11 +778,16 @@ void slow_hash_allocate_state(void) #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ defined(__DragonFly__) || defined(__NetBSD__) - hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANON, 0, 0); +#ifdef __NetBSD__ +#define RESERVED_FLAGS PROT_MPROTECT(PROT_EXEC) +#else +#define RESERVED_FLAGS 0 +#endif + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | RESERVED_FLAGS, + MAP_PRIVATE | MAP_ANON, -1, 0); #else - hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); #endif if(hp_jitfunc_memory == MAP_FAILED) hp_jitfunc_memory = NULL; @@ -794,9 +799,6 @@ void slow_hash_allocate_state(void) hp_jitfunc_memory = malloc(4096 + 4095); } hp_jitfunc = (v4_random_math_JIT_func)((size_t)(hp_jitfunc_memory + 4095) & ~4095); -#if !(defined(_MSC_VER) || defined(__MINGW32__)) - mprotect(hp_jitfunc, 4096, PROT_READ | PROT_WRITE | PROT_EXEC); -#endif } /** @@ -893,6 +895,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int if(hp_state == NULL) slow_hash_allocate_state(); + // locals to avoid constant TLS dereferencing + uint8_t *local_hp_state = hp_state; + v4_random_math_JIT_func local_hp_jitfunc = hp_jitfunc; + /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { memcpy(&state.hs, data, length); @@ -915,7 +921,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } else @@ -927,7 +933,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(j = 0; j < INIT_SIZE_BLK; j++) aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } @@ -975,7 +981,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } } else @@ -985,7 +991,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { for(j = 0; j < INIT_SIZE_BLK; j++) { - xor_blocks(&text[j * AES_BLOCK_SIZE], &hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + xor_blocks(&text[j * AES_BLOCK_SIZE], &local_hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); } } @@ -1065,24 +1071,24 @@ union cn_slow_hash_state #define pre_aes() \ j = state_index(a); \ - _c = vld1q_u8(&hp_state[j]); \ + _c = vld1q_u8(&local_hp_state[j]); \ _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ - VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ - VARIANT1_1(&hp_state[j]); \ + vst1q_u8(&local_hp_state[j], veorq_u8(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ - VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ @@ -1245,9 +1251,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int RDATA_ALIGN16 uint8_t expandedKey[240]; #ifndef FORCE_USE_HEAP - RDATA_ALIGN16 uint8_t hp_state[MEMORY]; + RDATA_ALIGN16 uint8_t local_hp_state[MEMORY]; #else - uint8_t *hp_state = (uint8_t *)aligned_malloc(MEMORY,16); + uint8_t *local_hp_state = (uint8_t *)aligned_malloc(MEMORY,16); #endif uint8_t text[INIT_SIZE_BYTE]; @@ -1287,7 +1293,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0]; @@ -1322,7 +1328,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } /* CryptoNight Step 5: Apply Keccak to the state again, and then @@ -1337,7 +1343,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int extra_hashes[state.hs.b[0] & 3](&state, 200, hash); #ifdef FORCE_USE_HEAP - aligned_free(hp_state); + aligned_free(local_hp_state); #endif } #else /* aarch64 && crypto */ diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 4ce9f3eeb..3dd98f0c6 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -40,6 +40,7 @@ #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> #include "cryptonote_basic.h" +#include "difficulty.h" #include "common/unordered_containers_boost_serialization.h" #include "crypto/crypto.h" #include "ringct/rctTypes.h" @@ -339,6 +340,41 @@ namespace boost if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2) a & x.p.pseudoOuts; } + + template <class Archive> + inline void serialize(Archive &a, rct::RCTConfig &x, const boost::serialization::version_type ver) + { + a & x.range_proof_type; + a & x.bp_version; + } + + template <class Archive> + inline void serialize(Archive &a, cryptonote::difficulty_type &x, const boost::serialization::version_type ver) + { + if (Archive::is_loading::value) + { + // load high part + uint64_t v = 0; + a & v; + x = v; + // load low part + x = x << 64; + a & v; + x += v; + } + else + { + // store high part + cryptonote::difficulty_type x_ = x >> 64; + uint64_t v = x_.convert_to<uint64_t>(); + a & v; + // store low part + x_ = x << 64 >> 64; + v = x_.convert_to<uint64_t>(); + a & v; + } + } + } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index f40464bd1..094057b1f 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -282,6 +282,11 @@ namespace cryptonote //--------------------------------------------------------------- bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { + if (hwdev.compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki)) + { + return true; + } + if (ack.m_spend_secret_key == crypto::null_skey) { // for watch-only wallet, simply copy the known output pubkey @@ -982,15 +987,11 @@ namespace cryptonote { if (t.version == 1) return false; - static const crypto::hash empty_hash = { (char)0x70, (char)0xa4, (char)0x85, (char)0x5d, (char)0x04, (char)0xd8, (char)0xfa, (char)0x7b, (char)0x3b, (char)0x27, (char)0x82, (char)0xca, (char)0x53, (char)0xb6, (char)0x00, (char)0xe5, (char)0xc0, (char)0x03, (char)0xc7, (char)0xdc, (char)0xb2, (char)0x7d, (char)0x7e, (char)0x92, (char)0x3c, (char)0x23, (char)0xf7, (char)0x86, (char)0x01, (char)0x46, (char)0xd2, (char)0xc5 }; const unsigned int unprunable_size = t.unprunable_size; if (blob && unprunable_size) { CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); - if (blob->size() - unprunable_size == 0) - res = empty_hash; - else - cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); } else { @@ -1002,10 +1003,7 @@ namespace cryptonote const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - if (ss.str().empty()) - res = empty_hash; - else - cryptonote::get_blob_hash(ss.str(), res); + cryptonote::get_blob_hash(ss.str(), res); } return true; } @@ -1042,7 +1040,10 @@ namespace cryptonote } // prunable rct - hashes[2] = pruned_data_hash; + if (t.rct_signatures.type == rct::RCTTypeNull) + hashes[2] = crypto::null_hash; + else + hashes[2] = pruned_data_hash; // the tx hash is the hash of the 3 hashes crypto::hash res = cn_fast_hash(hashes, sizeof(hashes)); diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index 89b748a83..5162e53e6 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -102,7 +102,7 @@ namespace cryptonote { return a + b < a || (c && a + b == (uint64_t) -1); } - bool check_hash(const crypto::hash &hash, difficulty_type difficulty) { + bool check_hash_64(const crypto::hash &hash, uint64_t difficulty) { uint64_t low, high, top, cur; // First check the highest word, this will most likely fail for a random hash. mul(swap64le(((const uint64_t *) &hash)[3]), difficulty, top, high); @@ -119,7 +119,7 @@ namespace cryptonote { return !carry; } - difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) { + uint64_t next_difficulty_64(std::vector<std::uint64_t> timestamps, std::vector<uint64_t> cumulative_difficulties, size_t target_seconds) { if(timestamps.size() > DIFFICULTY_WINDOW) { @@ -150,7 +150,7 @@ namespace cryptonote { if (time_span == 0) { time_span = 1; } - difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; + uint64_t total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; assert(total_work > 0); uint64_t low, high; mul(total_work, target_seconds, low, high); @@ -162,4 +162,81 @@ namespace cryptonote { return (low + time_span - 1) / time_span; } +#if defined(_MSC_VER) +#ifdef max +#undef max +#endif +#endif + + const difficulty_type max64bit(std::numeric_limits<std::uint64_t>::max()); + const boost::multiprecision::uint256_t max128bit(std::numeric_limits<boost::multiprecision::uint128_t>::max()); + const boost::multiprecision::uint512_t max256bit(std::numeric_limits<boost::multiprecision::uint256_t>::max()); + +#define FORCE_FULL_128_BITS + + bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty) { +#ifndef FORCE_FULL_128_BITS + // fast check + if (difficulty >= max64bit && ((const uint64_t *) &hash)[3] > 0) + return false; +#endif + // usual slow check + boost::multiprecision::uint512_t hashVal = 0; +#ifdef FORCE_FULL_128_BITS + for(int i = 0; i < 4; i++) { // highest word is zero +#else + for(int i = 1; i < 4; i++) { // highest word is zero +#endif + hashVal <<= 64; + hashVal |= swap64le(((const uint64_t *) &hash)[3 - i]); + } + return hashVal * difficulty <= max256bit; + } + + bool check_hash(const crypto::hash &hash, difficulty_type difficulty) { + if (difficulty <= max64bit) // if can convert to small difficulty - do it + return check_hash_64(hash, difficulty.convert_to<std::uint64_t>()); + else + return check_hash_128(hash, difficulty); + } + + difficulty_type next_difficulty(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) { + //cutoff DIFFICULTY_LAG + if(timestamps.size() > DIFFICULTY_WINDOW) + { + timestamps.resize(DIFFICULTY_WINDOW); + cumulative_difficulties.resize(DIFFICULTY_WINDOW); + } + + + size_t length = timestamps.size(); + assert(length == cumulative_difficulties.size()); + if (length <= 1) { + return 1; + } + static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small"); + assert(length <= DIFFICULTY_WINDOW); + sort(timestamps.begin(), timestamps.end()); + size_t cut_begin, cut_end; + static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large"); + if (length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) { + cut_begin = 0; + cut_end = length; + } else { + cut_begin = (length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + 1) / 2; + cut_end = cut_begin + (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT); + } + assert(/*cut_begin >= 0 &&*/ cut_begin + 2 <= cut_end && cut_end <= length); + uint64_t time_span = timestamps[cut_end - 1] - timestamps[cut_begin]; + if (time_span == 0) { + time_span = 1; + } + difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; + assert(total_work > 0); + boost::multiprecision::uint256_t res = (boost::multiprecision::uint256_t(total_work) * target_seconds + time_span - 1) / time_span; + if(res > max128bit) + return 0; // to behave like previous implementation, may be better return max128bit? + return res.convert_to<difficulty_type>(); + } + } diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index 8da355b22..f7a9376fb 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -32,12 +32,13 @@ #include <cstdint> #include <vector> +#include <boost/multiprecision/cpp_int.hpp> #include "crypto/hash.h" namespace cryptonote { - typedef std::uint64_t difficulty_type; + typedef boost::multiprecision::uint128_t difficulty_type; /** * @brief checks if a hash fits the given difficulty @@ -51,6 +52,10 @@ namespace cryptonote * * @return true if valid, else false */ + bool check_hash_64(const crypto::hash &hash, uint64_t difficulty); + uint64_t next_difficulty_64(std::vector<std::uint64_t> timestamps, std::vector<uint64_t> cumulative_difficulties, size_t target_seconds); + + bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty); bool check_hash(const crypto::hash &hash, difficulty_type difficulty); difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds); } diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index ebfe45f37..89bca2f09 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -317,9 +317,12 @@ void HardFork::on_block_popped(uint64_t nblocks) uint64_t height; for (height = old_chain_height - 1; height >= new_chain_height; --height) { + version = versions.back(); + last_versions[version]--; versions.pop_back(); version = db.get_hard_fork_version(height); versions.push_front(version); + last_versions[version]++; } // does not take voting into account diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 6628c8448..4e2edc20f 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -435,14 +435,15 @@ namespace cryptonote { MTRACE("Miner has received stop signal"); - if (!is_mining()) + CRITICAL_REGION_LOCAL(m_threads_lock); + bool mining = !m_threads.empty(); + if (!mining) { MTRACE("Not mining - nothing to stop" ); return true; } send_stop_signal(); - CRITICAL_REGION_LOCAL(m_threads_lock); // In case background mining was active and the miner threads are waiting // on the background miner to signal start. diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 3e903cd12..263227148 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -645,6 +645,8 @@ block Blockchain::pop_block_from_blockchain() block popped_block; std::vector<transaction> popped_txs; + CHECK_AND_ASSERT_THROW_MES(m_db->height() > 1, "Cannot pop the genesis block"); + try { m_db->pop_block(popped_block, popped_txs); @@ -1269,15 +1271,17 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_ if(h == 0) return; - m_db->block_txn_start(true); // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); - weights.reserve(weights.size() + h - start_offset); - for(size_t i = start_offset; i < h; i++) - { - weights.push_back(m_db->get_block_weight(i)); - } - m_db->block_txn_stop(); + weights = m_db->get_block_weights(start_offset, count); +} +//------------------------------------------------------------------ +void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + weights = m_db->get_long_term_block_weights(start_height, count); } //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_block_weight_limit() const @@ -1995,7 +1999,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc return true; } //------------------------------------------------------------------ -uint64_t Blockchain::block_difficulty(uint64_t i) const +difficulty_type Blockchain::block_difficulty(uint64_t i) const { LOG_PRINT_L3("Blockchain::" << __func__); // WARNING: this function does not take m_blockchain_lock, and thus should only call read only @@ -2194,7 +2198,11 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height); if (result) - resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); + { + cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); + resp.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + resp.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); + } return result; } @@ -3804,9 +3812,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons return block_weight; std::vector<uint64_t> weights; - weights.resize(nblocks); - for (uint64_t h = 0; h < nblocks; ++h) - weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h); + get_long_term_block_weights(weights, db_height - nblocks, nblocks); uint64_t long_term_median = epee::misc_utils::median(weights); uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); @@ -3850,9 +3856,7 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); if (nblocks == db_height) --nblocks; - weights.resize(nblocks); - for (uint64_t h = 0; h < nblocks; ++h) - weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1); + get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks); new_weights = weights; long_term_median = epee::misc_utils::median(weights); } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 4744defc5..89d8e7572 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -653,7 +653,7 @@ namespace cryptonote * * @return the difficulty */ - uint64_t block_difficulty(uint64_t i) const; + difficulty_type block_difficulty(uint64_t i) const; /** * @brief gets blocks based on a list of block hashes @@ -1288,14 +1288,25 @@ namespace cryptonote /** * @brief gets recent block weights for median calculation * - * get the block weights of the last <count> blocks, and return by reference <sz>. + * get the block weights of the last <count> blocks, and return by reference <weights>. * - * @param sz return-by-reference the list of weights + * @param weights return-by-reference the list of weights * @param count the number of blocks to get weights for */ void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const; /** + * @brief gets recent block long term weights for median calculation + * + * get the block long term weights of the last <count> blocks, and return by reference <weights>. + * + * @param weights return-by-reference the list of weights + * @param start_height the block height of the first block to query + * @param count the number of blocks to get weights for + */ + void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const; + + /** * @brief checks if a transaction is unlocked (its outputs spendable) * * This function checks to see if a transaction is unlocked. diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 58acdb6bb..da413bbe2 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -921,6 +921,7 @@ namespace cryptonote catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_pre: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); @@ -951,6 +952,7 @@ namespace cryptonote catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_post: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); @@ -1782,12 +1784,28 @@ namespace cryptonote return f; } //----------------------------------------------------------------------------------------------- - static double probability(unsigned int blocks, unsigned int expected) + static double probability1(unsigned int blocks, unsigned int expected) { // https://www.umass.edu/wsp/resources/poisson/#computing return pow(expected, blocks) / (factorial(blocks) * exp(expected)); } //----------------------------------------------------------------------------------------------- + static double probability(unsigned int blocks, unsigned int expected) + { + double p = 0.0; + if (blocks <= expected) + { + for (unsigned int b = 0; b <= blocks; ++b) + p += probability1(b, expected); + } + else if (blocks > expected) + { + for (unsigned int b = blocks; b <= expected * 3 /* close enough */; ++b) + p += probability1(b, expected); + } + return p; + } + //----------------------------------------------------------------------------------------------- bool core::check_block_rate() { if (m_offline || m_target_blockchain_height > get_current_blockchain_height()) diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index d582e3e9c..3083a5b4c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -34,6 +34,7 @@ #include "serialization/keyvalue_serialization.h" #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/blobdatatype.h" + namespace cryptonote { @@ -208,6 +209,7 @@ namespace cryptonote { uint64_t current_height; uint64_t cumulative_difficulty; + uint64_t cumulative_difficulty_top64; crypto::hash top_id; uint8_t top_version; uint32_t pruning_seed; @@ -215,6 +217,7 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(current_height) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_VAL_POD_AS_BLOB(top_id) KV_SERIALIZE_OPT(top_version, (uint8_t)0) KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) @@ -245,12 +248,14 @@ namespace cryptonote uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; + uint64_t cumulative_difficulty_top64; std::vector<crypto::hash> m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) KV_SERIALIZE(total_height) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index b33867e8b..32f0afceb 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -398,7 +398,9 @@ namespace cryptonote { m_core.get_blockchain_top(hshd.current_height, hshd.top_id); hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); - hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); + difficulty_type wide_cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); + hshd.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + hshd.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); hshd.current_height +=1; hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; diff --git a/src/device/device.hpp b/src/device/device.hpp index 7f1a60c9f..65b38361b 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -70,6 +70,7 @@ namespace cryptonote struct account_keys; struct subaddress_index; struct tx_destination_entry; + struct keypair; } namespace hw { @@ -81,11 +82,18 @@ namespace hw { return false; } + class device_progress { + public: + virtual double progress() const { return 0; } + virtual bool indeterminate() const { return false; } + }; + class i_device_callback { public: - virtual void on_button_request() {} - virtual void on_pin_request(epee::wipeable_string & pin) {} - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual void on_button_request(uint64_t code=0) {} + virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; } + virtual void on_progress(const device_progress& event) {} virtual ~i_device_callback() = default; }; @@ -141,6 +149,9 @@ namespace hw { virtual void set_callback(i_device_callback * callback) {}; virtual void set_derivation_path(const std::string &derivation_path) {}; + virtual void set_pin(const epee::wipeable_string & pin) {} + virtual void set_passphrase(const epee::wipeable_string & passphrase) {} + /* ======================================================================= */ /* LOCKER */ /* ======================================================================= */ @@ -229,7 +240,9 @@ namespace hw { virtual bool has_ki_cold_sync(void) const { return false; } virtual bool has_tx_cold_sign(void) const { return false; } - + virtual bool has_ki_live_refresh(void) const { return true; } + virtual bool compute_key_image(const cryptonote::account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const cryptonote::subaddress_index& received_index, cryptonote::keypair& in_ephemeral, crypto::key_image& ki) { return false; } + virtual void computing_key_images(bool started) {}; virtual void set_network_type(cryptonote::network_type network_type) { } protected: diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index 8b8cdf6d2..31b1504ab 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -31,6 +31,7 @@ #define MONERO_DEVICE_COLD_H #include "wallet/wallet2.h" +#include <boost/optional/optional.hpp> #include <boost/function.hpp> @@ -44,6 +45,8 @@ namespace hw { public: std::vector<std::string> tx_device_aux; // device generated aux data std::vector<cryptonote::address_parse_info> tx_recipients; // as entered by user + boost::optional<int> bp_version; // BP version to use + boost::optional<unsigned> client_version; // Signing client version to use (testing) }; class device_cold { @@ -51,6 +54,53 @@ namespace hw { using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>; + class op_progress : public hw::device_progress { + public: + op_progress():m_progress(0), m_indeterminate(false) {}; + explicit op_progress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate){} + + double progress() const override { return m_progress; } + bool indeterminate() const override { return m_indeterminate; } + protected: + double m_progress; + bool m_indeterminate; + }; + + class tx_progress : public op_progress { + public: + tx_progress(): + m_cur_tx(0), m_max_tx(1), + m_cur_step(0), m_max_step(1), + m_cur_substep(0), m_max_substep(1){}; + + tx_progress(size_t cur_tx, size_t max_tx, size_t cur_step, size_t max_step, size_t cur_substep, size_t max_substep): + m_cur_tx(cur_tx), m_max_tx(max_tx), + m_cur_step(cur_tx), m_max_step(max_tx), + m_cur_substep(cur_tx), m_max_substep(max_tx){} + + double progress() const override { + return std::max(1.0, (double)m_cur_tx / m_max_tx + + (double)m_cur_step / (m_max_tx * m_max_step) + + (double)m_cur_substep / (m_max_tx * m_max_step * m_max_substep)); + } + bool indeterminate() const override { return false; } + + protected: + size_t m_cur_tx; + size_t m_max_tx; + size_t m_cur_step; + size_t m_max_step; + size_t m_cur_substep; + size_t m_max_substep; + }; + + typedef struct { + std::string salt1; + std::string salt2; + std::string tx_enc_keys; + std::string tx_prefix_hash; + } tx_key_data_t; + /** * Key image sync with the cold protocol. */ @@ -65,6 +115,52 @@ namespace hw { const ::tools::wallet2::unsigned_tx_set & unsigned_tx, ::tools::wallet2::signed_tx_set & signed_tx, tx_aux_data & aux_data) =0; + + /** + * Get tx key support check. + */ + virtual bool is_get_tx_key_supported() const { return false; } + + /** + * Loads TX aux data required for tx key. + */ + virtual void load_tx_key_data(tx_key_data_t & res, const std::string & tx_aux_data) =0; + + /** + * Decrypts TX keys. + */ + virtual void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) =0; + + /** + * Live refresh support check + */ + virtual bool is_live_refresh_supported() const { return false; }; + + /** + * Starts live refresh process with the device + */ + virtual void live_refresh_start() =0; + + /** + * One live refresh step + */ + virtual void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) =0; + + /** + * Live refresh process termination + */ + virtual void live_refresh_finish() =0; }; } diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index ceb6111e0..b4a80cf2c 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -57,7 +57,9 @@ namespace trezor { } device_trezor::device_trezor() { - + m_live_refresh_in_progress = false; + m_live_refresh_enabled = true; + m_live_refresh_thread_running = false; } device_trezor::~device_trezor() { @@ -69,6 +71,89 @@ namespace trezor { } } + bool device_trezor::init() + { + m_live_refresh_in_progress = false; + bool r = device_trezor_base::init(); + if (r && !m_live_refresh_thread) + { + m_live_refresh_thread_running = true; + m_live_refresh_thread.reset(new boost::thread(boost::bind(&device_trezor::live_refresh_thread_main, this))); + } + return r; + } + + bool device_trezor::release() + { + m_live_refresh_in_progress = false; + m_live_refresh_thread_running = false; + if (m_live_refresh_thread) + { + m_live_refresh_thread->join(); + m_live_refresh_thread = nullptr; + } + return device_trezor_base::release(); + } + + bool device_trezor::disconnect() + { + m_live_refresh_in_progress = false; + return device_trezor_base::disconnect(); + } + + void device_trezor::device_state_reset_unsafe() + { + require_connected(); + if (m_live_refresh_in_progress) + { + try + { + live_refresh_finish_unsafe(); + } + catch(const std::exception & e) + { + MERROR("Live refresh could not be terminated: " << e.what()); + } + } + + m_live_refresh_in_progress = false; + device_trezor_base::device_state_reset_unsafe(); + } + + void device_trezor::live_refresh_thread_main() + { + while(m_live_refresh_thread_running) + { + boost::this_thread::sleep_for(boost::chrono::milliseconds(100)); + if (!m_live_refresh_in_progress) + { + continue; + } + + TREZOR_AUTO_LOCK_DEVICE(); + if (!m_transport || !m_live_refresh_in_progress) + { + continue; + } + + auto current_time = std::chrono::steady_clock::now(); + if (current_time - m_last_live_refresh_time <= std::chrono::seconds(20)) + { + continue; + } + + MTRACE("Closing live refresh process due to inactivity"); + try + { + live_refresh_finish(); + } + catch(const std::exception &e) + { + MWARNING("Live refresh auto-finish failed: " << e.what()); + } + } + } + /* ======================================================================= */ /* WALLET & ADDRESS */ /* ======================================================================= */ @@ -126,7 +211,7 @@ namespace trezor { std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address( const boost::optional<std::vector<uint32_t>> & path, const boost::optional<cryptonote::network_type> & network_type){ - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -142,7 +227,7 @@ namespace trezor { std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key( const boost::optional<std::vector<uint32_t>> & path, const boost::optional<cryptonote::network_type> & network_type){ - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -155,11 +240,43 @@ namespace trezor { return response; } + bool device_trezor::is_get_tx_key_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); + } + + void device_trezor::load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) + { + protocol::tx::load_tx_key_data(res, tx_aux_data); + } + + void device_trezor::get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = protocol::tx::get_tx_key(tx_aux_data); + this->set_msg_addr<messages::monero::MoneroGetTxKeyRequest>(req.get()); + + auto response = this->client_exchange<messages::monero::MoneroGetTxKeyAck>(req); + MTRACE("Get TX key response received"); + + protocol::tx::get_tx_key_ack(tx_keys, tx_aux_data.tx_prefix_hash, view_key_priv, response); + } + void device_trezor::ki_sync(wallet_shim * wallet, const std::vector<tools::wallet2::transfer_details> & transfers, hw::device_cold::exported_key_image & ski) { - AUTO_LOCK_CMD(); +#define EVENT_PROGRESS(P) do { if (m_callback) {(m_callback)->on_progress(device_cold::op_progress(P)); } }while(0) + + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -171,6 +288,7 @@ namespace trezor { protocol::ki::key_image_data(wallet, transfers, mtds); protocol::ki::generate_commitment(mtds, transfers, req); + EVENT_PROGRESS(0.); this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req); @@ -194,27 +312,160 @@ namespace trezor { } MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + EVENT_PROGRESS((double)cur * batch_size / mtds.size()); } + EVENT_PROGRESS(1.); auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>(); auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req); ski.reserve(kis.size()); for(auto & sub : kis){ - char buff[32*3]; + ::crypto::signature sig{}; + ::crypto::key_image ki; + char buff[sizeof(ki.data)*3]; + + size_t buff_len = sizeof(buff); + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()), - reinterpret_cast<const uint8_t *>(sub.iv().data()), buff); + reinterpret_cast<const uint8_t *>(sub.iv().data()), buff, &buff_len); + CHECK_AND_ASSERT_THROW_MES(buff_len == sizeof(buff), "Plaintext size invalid"); - ::crypto::signature sig{}; - ::crypto::key_image ki; - memcpy(ki.data, buff, 32); - memcpy(sig.c.data, buff + 32, 32); - memcpy(sig.r.data, buff + 64, 32); + memcpy(ki.data, buff, sizeof(ki.data)); + memcpy(sig.c.data, buff + sizeof(ki.data), sizeof(ki.data)); + memcpy(sig.r.data, buff + 2*sizeof(ki.data), sizeof(ki.data)); ski.push_back(std::make_pair(ki, sig)); } +#undef EVENT_PROGRESS + } + + bool device_trezor::is_live_refresh_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); } + bool device_trezor::is_live_refresh_enabled() const + { + return is_live_refresh_supported() && (mode == NONE || mode == TRANSACTION_PARSE) && m_live_refresh_enabled; + } + + bool device_trezor::has_ki_live_refresh() const + { + try{ + return is_live_refresh_enabled(); + } catch(const std::exception & e){ + MERROR("Could not detect if live refresh is enabled: " << e.what()); + } + return false; + } + + void device_trezor::live_refresh_start() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + live_refresh_start_unsafe(); + } + + void device_trezor::live_refresh_start_unsafe() + { + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>(); + this->set_msg_addr<messages::monero::MoneroLiveRefreshStartRequest>(req.get()); + this->client_exchange<messages::monero::MoneroLiveRefreshStartAck>(req); + m_live_refresh_in_progress = true; + m_last_live_refresh_time = std::chrono::steady_clock::now(); + } + + void device_trezor::live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + + if (!m_live_refresh_in_progress) + { + live_refresh_start_unsafe(); + } + + m_last_live_refresh_time = std::chrono::steady_clock::now(); + + auto req = std::make_shared<messages::monero::MoneroLiveRefreshStepRequest>(); + req->set_out_key(out_key.data, 32); + req->set_recv_deriv(recv_derivation.data, 32); + req->set_real_out_idx(real_output_index); + req->set_sub_addr_major(received_index.major); + req->set_sub_addr_minor(received_index.minor); + + auto ack = this->client_exchange<messages::monero::MoneroLiveRefreshStepAck>(req); + protocol::ki::live_refresh_ack(view_key_priv, out_key, ack, in_ephemeral, ki); + } + + void device_trezor::live_refresh_finish_unsafe() + { + auto req = std::make_shared<messages::monero::MoneroLiveRefreshFinalRequest>(); + this->client_exchange<messages::monero::MoneroLiveRefreshFinalAck>(req); + m_live_refresh_in_progress = false; + } + + void device_trezor::live_refresh_finish() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + if (m_live_refresh_in_progress) + { + live_refresh_finish_unsafe(); + } + } + + void device_trezor::computing_key_images(bool started) + { + try + { + if (!is_live_refresh_enabled()) + { + return; + } + + // React only on termination as the process can auto-start itself. + if (!started && m_live_refresh_in_progress) + { + live_refresh_finish(); + } + } + catch(const std::exception & e) + { + MWARNING("KI computation state change failed, started: " << started << ", e: " << e.what()); + } + } + + bool device_trezor::compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + if (!is_live_refresh_enabled()) + { + return false; + } + + live_refresh(ack.m_view_secret_key, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki); + return true; + } void device_trezor::tx_sign(wallet_shim * wallet, const tools::wallet2::unsigned_tx_set & unsigned_tx, @@ -222,7 +473,15 @@ namespace trezor { hw::tx_aux_data & aux_data) { CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); - size_t num_tx = unsigned_tx.txes.size(); + + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + transaction_versions_check(unsigned_tx, aux_data); + + const size_t num_tx = unsigned_tx.txes.size(); + m_num_transations_to_sign = num_tx; signed_tx.key_images.clear(); signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); @@ -267,6 +526,10 @@ namespace trezor { cpend.key_images = key_images; // KI sync + for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){ + signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image; + } + size_t num_sources = cdata.tx_data.sources.size(); CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); @@ -276,12 +539,19 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; - auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); + CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset"); + idx_map_src -= unsigned_tx.transfers.first; CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + + const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); signed_tx.key_images[idx_map_src] = vini.k_image; } } + + if (m_callback){ + m_callback->on_progress(device_cold::tx_progress(m_num_transations_to_sign, m_num_transations_to_sign, 1, 1, 1, 1)); + } } void device_trezor::tx_sign(wallet_shim * wallet, @@ -290,10 +560,16 @@ namespace trezor { hw::tx_aux_data & aux_data, std::shared_ptr<protocol::tx::Signer> & signer) { - AUTO_LOCK_CMD(); +#define EVENT_PROGRESS(S, SUB, SUBMAX) do { if (m_callback) { \ + (m_callback)->on_progress(device_cold::tx_progress(idx, m_num_transations_to_sign, S, 10, SUB, SUBMAX)); \ +} }while(0) + require_connected(); - device_state_reset_unsafe(); + if (idx > 0) + device_state_reset_unsafe(); + require_initialized(); + EVENT_PROGRESS(0, 1, 1); CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data); @@ -305,6 +581,7 @@ namespace trezor { auto init_msg = signer->step_init(); this->set_msg_addr(init_msg.get()); transaction_pre_check(init_msg); + EVENT_PROGRESS(1, 1, 1); auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg); signer->step_init_ack(response); @@ -314,6 +591,7 @@ namespace trezor { auto src = signer->step_set_input(cur_src); auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src); signer->step_set_input_ack(ack); + EVENT_PROGRESS(2, cur_src, num_sources); } // Step: sort @@ -322,44 +600,82 @@ namespace trezor { auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req); signer->step_permutation_ack(perm_ack); } + EVENT_PROGRESS(3, 1, 1); // Step: input_vini - if (!signer->in_memory()){ - for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ - auto src = signer->step_set_vini_input(cur_src); - auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); - signer->step_set_vini_input_ack(ack); - } + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); + signer->step_set_vini_input_ack(ack); + EVENT_PROGRESS(4, cur_src, num_sources); } // Step: all inputs set auto all_inputs_set = signer->step_all_inputs_set(); auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set); signer->step_all_inputs_set_ack(ack_all_inputs); + EVENT_PROGRESS(5, 1, 1); // Step: outputs for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ auto src = signer->step_set_output(cur_dst); auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src); signer->step_set_output_ack(ack); + + // If BP is offloaded to host, another step with computed BP may be needed. + auto offloaded_bp = signer->step_rsig(cur_dst); + if (offloaded_bp){ + auto bp_ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(offloaded_bp); + signer->step_set_rsig_ack(ack); + } + + EVENT_PROGRESS(6, cur_dst, num_outputs); } // Step: all outs set auto all_out_set = signer->step_all_outs_set(); auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set); signer->step_all_outs_set_ack(ack_all_out_set, *this); + EVENT_PROGRESS(7, 1, 1); // Step: sign each input for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ auto src = signer->step_sign_input(cur_src); auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src); signer->step_sign_input_ack(ack_sign); + EVENT_PROGRESS(8, cur_src, num_sources); } // Step: final auto final_msg = signer->step_final(); auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg); signer->step_final_ack(ack_final); + EVENT_PROGRESS(9, 1, 1); +#undef EVENT_PROGRESS + } + + void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data) + { + auto trezor_version = get_version(); + unsigned client_version = 1; // default client version for tx + + if (trezor_version <= pack_version(2, 0, 10)){ + client_version = 0; + } + + if (aux_data.client_version){ + auto wanted_client_version = aux_data.client_version.get(); + if (wanted_client_version > client_version){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update."); + } else { + client_version = wanted_client_version; + } + } + aux_data.client_version = client_version; + + if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update."); + } } void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg) diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 75f6d5875..0e91847dc 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -30,18 +30,21 @@ #ifndef MONERO_DEVICE_TREZOR_H #define MONERO_DEVICE_TREZOR_H +#include "trezor.hpp" +#include "device/device.hpp" +#ifdef WITH_DEVICE_TREZOR #include <cstddef> #include <string> -#include "device/device.hpp" -#include "device/device_default.hpp" -#include "device/device_cold.hpp" #include <boost/scope_exit.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> + +#include "device/device_default.hpp" +#include "device/device_cold.hpp" #include "cryptonote_config.h" -#include "trezor.hpp" #include "device_trezor_base.hpp" +#endif namespace hw { namespace trezor { @@ -57,8 +60,29 @@ namespace trezor { */ class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { protected: + std::atomic<bool> m_live_refresh_in_progress; + std::chrono::steady_clock::time_point m_last_live_refresh_time; + std::unique_ptr<boost::thread> m_live_refresh_thread; + std::atomic<bool> m_live_refresh_thread_running; + bool m_live_refresh_enabled; + size_t m_num_transations_to_sign; + + void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data); void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg); void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); + void device_state_reset_unsafe() override; + void live_refresh_start_unsafe(); + void live_refresh_finish_unsafe(); + void live_refresh_thread_main(); + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer); public: device_trezor(); @@ -69,11 +93,17 @@ namespace trezor { explicit operator bool() const override {return true;} + bool init() override; + bool release() override; + bool disconnect() override; + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; bool has_ki_cold_sync() const override { return true; } bool has_tx_cold_sign() const override { return true; } void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + void set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; } + bool live_refresh_enabled() const { return m_live_refresh_enabled; } /* ======================================================================= */ /* WALLET & ADDRESS */ @@ -100,20 +130,68 @@ namespace trezor { const boost::optional<cryptonote::network_type> & network_type = boost::none); /** + * Get_tx_key support check + */ + bool is_get_tx_key_supported() const override; + + /** + * Loads tx aux data + */ + void load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) override; + + /** + * TX key load with the Trezor + */ + void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) override; + + /** * Key image sync with the Trezor. */ void ki_sync(wallet_shim * wallet, const std::vector<::tools::wallet2::transfer_details> & transfers, hw::device_cold::exported_key_image & ski) override; + bool is_live_refresh_supported() const override; + + bool is_live_refresh_enabled() const; + + bool has_ki_live_refresh() const override; + + void live_refresh_start() override; + + void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) override; + + void live_refresh_finish() override; + /** - * Signs particular transaction idx in the unsigned set, keeps state in the signer + * Letting device know the KI computation started / ended. + * During refresh */ - void tx_sign(wallet_shim * wallet, - const ::tools::wallet2::unsigned_tx_set & unsigned_tx, - size_t idx, - hw::tx_aux_data & aux_data, - std::shared_ptr<protocol::tx::Signer> & signer); + void computing_key_images(bool started) override; + + /** + * Implements hw::device interface + * called from generate_key_image_helper_precomp() + */ + bool compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) override; /** * Signs unsigned transaction with the Trezor. diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 1892b47d9..f3d15c5e2 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -109,6 +109,7 @@ namespace trezor { disconnect(); // Enumerate all available devices + TREZOR_AUTO_LOCK_DEVICE(); try { hw::trezor::t_transport_vect trans; @@ -145,6 +146,7 @@ namespace trezor { } bool device_trezor_base::disconnect() { + TREZOR_AUTO_LOCK_DEVICE(); m_device_state.clear(); m_features.reset(); @@ -203,13 +205,13 @@ namespace trezor { /* Helpers */ /* ======================================================================= */ - void device_trezor_base::require_connected(){ + void device_trezor_base::require_connected() const { if (!m_transport){ throw exc::NotConnectedException(); } } - void device_trezor_base::require_initialized(){ + void device_trezor_base::require_initialized() const { if (!m_features){ throw exc::TrezorException("Device state not initialized"); } @@ -330,7 +332,7 @@ namespace trezor { /* ======================================================================= */ bool device_trezor_base::ping() { - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); if (!m_transport){ MINFO("Ping failed, device not connected"); return false; @@ -364,7 +366,7 @@ namespace trezor { void device_trezor_base::device_state_reset() { - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); device_state_reset_unsafe(); } @@ -373,6 +375,10 @@ namespace trezor { if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \ if (m_callback) m_callback->method(__VA_ARGS__); \ }while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) do { \ + if (m_debug_callback) VAR = m_debug_callback->method(__VA_ARGS__); \ + if (m_callback) VAR = m_callback->method(__VA_ARGS__); \ +}while(0) void device_trezor_base::setup_debug(){ if (!m_debug){ @@ -392,6 +398,7 @@ namespace trezor { #else #define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) VAR = (m_callback ? m_callback->method(__VA_ARGS__) : boost::none) #endif void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) @@ -402,7 +409,7 @@ namespace trezor { messages::common::ButtonAck ack; write_raw(&ack); - TREZOR_CALLBACK(on_button_request); + TREZOR_CALLBACK(on_button_request, msg->code()); resp = read_raw(); } @@ -411,13 +418,18 @@ namespace trezor { MDEBUG("on_pin_request"); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - epee::wipeable_string pin; + boost::optional<epee::wipeable_string> pin; + TREZOR_CALLBACK_GET(pin, on_pin_request); - TREZOR_CALLBACK(on_pin_request, pin); + if (!pin && m_pin){ + pin = m_pin; + } // TODO: remove PIN from memory messages::common::PinMatrixAck m; - m.set_pin(pin.data(), pin.size()); + if (pin) { + m.set_pin(pin.get().data(), pin.get().size()); + } resp = call_raw(&m); } @@ -425,14 +437,19 @@ namespace trezor { { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); MDEBUG("on_passhprase_request, on device: " << msg->on_device()); - epee::wipeable_string passphrase; + boost::optional<epee::wipeable_string> passphrase; + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device()); - TREZOR_CALLBACK(on_passphrase_request, msg->on_device(), passphrase); + if (!passphrase && m_passphrase){ + passphrase = m_passphrase; + } + + m_passphrase = boost::none; messages::common::PassphraseAck m; - if (!msg->on_device()){ + if (!msg->on_device() && passphrase){ // TODO: remove passphrase from memory - m.set_passphrase(passphrase.data(), passphrase.size()); + m.set_passphrase(passphrase.get().data(), passphrase.get().size()); } if (!m_device_state.empty()){ @@ -494,16 +511,16 @@ namespace trezor { m_debug_link->init(debug_transport); } - void trezor_debug_callback::on_button_request() { + void trezor_debug_callback::on_button_request(uint64_t code) { if (m_debug_link) m_debug_link->press_yes(); } - void trezor_debug_callback::on_pin_request(epee::wipeable_string &pin) { - + boost::optional<epee::wipeable_string> trezor_debug_callback::on_pin_request() { + return boost::none; } - void trezor_debug_callback::on_passphrase_request(bool on_device, epee::wipeable_string &passphrase) { - + boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) { + return boost::none; } void trezor_debug_callback::on_passphrase_state_request(const std::string &state) { diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 8942df797..8c3c14b29 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -47,14 +47,15 @@ #endif //automatic lock one more level on device ensuring the current thread is allowed to use it -#define AUTO_LOCK_CMD() \ +#define TREZOR_AUTO_LOCK_CMD() \ /* lock both mutexes without deadlock*/ \ boost::lock(device_locker, command_locker); \ /* make sure both already-locked mutexes are unlocked at the end of scope */ \ boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) - +#define TREZOR_AUTO_LOCK_DEVICE() boost::lock_guard<boost::recursive_mutex> lock1_device(device_locker) + namespace hw { namespace trezor { @@ -62,14 +63,14 @@ namespace trezor { class device_trezor_base; #ifdef WITH_TREZOR_DEBUGGING - class trezor_debug_callback { + class trezor_debug_callback : public hw::i_device_callback { public: trezor_debug_callback()=default; explicit trezor_debug_callback(std::shared_ptr<Transport> & debug_transport); - void on_button_request(); - void on_pin_request(epee::wipeable_string &pin); - void on_passphrase_request(bool on_device, epee::wipeable_string &passphrase); + void on_button_request(uint64_t code=0) override; + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; void on_passphrase_state_request(const std::string &state); void on_disconnect(); protected: @@ -95,6 +96,8 @@ namespace trezor { std::vector<unsigned int> m_wallet_deriv_path; std::string m_device_state; // returned after passphrase entry, session std::shared_ptr<messages::management::Features> m_features; // features from the last device reset + boost::optional<epee::wipeable_string> m_pin; + boost::optional<epee::wipeable_string> m_passphrase; cryptonote::network_type network_type; @@ -109,11 +112,11 @@ namespace trezor { // Internal methods // - void require_connected(); - void require_initialized(); + void require_connected() const; + void require_initialized() const; void call_ping_unsafe(); void test_ping(); - void device_state_reset_unsafe(); + virtual void device_state_reset_unsafe(); void ensure_derivation_path() noexcept; // Communication methods @@ -265,6 +268,15 @@ namespace trezor { void set_derivation_path(const std::string &deriv_path) override; + virtual bool has_ki_live_refresh(void) const override { return false; } + + virtual void set_pin(const epee::wipeable_string & pin) override { + m_pin = pin; + } + virtual void set_passphrase(const epee::wipeable_string & passphrase) override { + m_passphrase = passphrase; + } + /* ======================================================================= */ /* SETUP/TEARDOWN */ /* ======================================================================= */ diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 79f6adac9..5fe08abbe 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -33,6 +33,8 @@ #include <utility> #include <boost/endian/conversion.hpp> #include <common/apply_permutation.h> +#include <common/json_util.h> +#include <crypto/hmac-keccak.h> #include <ringct/rctSigs.h> #include <ringct/bulletproofs.h> #include "cryptonote_config.h" @@ -40,6 +42,37 @@ #include <sodium/crypto_verify_32.h> #include <sodium/crypto_aead_chacha20poly1305.h> +#define GET_FIELD_STRING(name, type, jtype) field_##name = std::string(json[#name].GetString(), json[#name].GetStringLength()) +#define GET_FIELD_OTHER(name, type, jtype) field_##name = static_cast<type>(json[#name].Get##jtype()) + +#define GET_STRING_FROM_JSON(json, name, type, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, String, mandatory, def, GET_FIELD_STRING) + +#define GET_FIELD_FROM_JSON(json, name, type, jtype, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, GET_FIELD_OTHER) + +#define GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, VAL) \ + type field_##name = static_cast<type>(def); \ + bool field_##name##_found = false; \ + (void)field_##name##_found; \ + do if (json.HasMember(#name)) \ + { \ + if (json[#name].Is##jtype()) \ + { \ + VAL(name, type, jtype); \ + field_##name##_found = true; \ + } \ + else \ + { \ + throw std::invalid_argument("Field " #name " found in JSON, but not " #jtype); \ + } \ + } \ + else if (mandatory) \ + { \ + throw std::invalid_argument("Field " #name " not found in JSON");\ + } while(0) + + namespace hw{ namespace trezor{ namespace protocol{ @@ -84,19 +117,22 @@ namespace protocol{ namespace crypto { namespace chacha { - void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){ - if (length < 16){ - throw std::invalid_argument("Ciphertext length too small"); - } + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len){ + CHECK_AND_ASSERT_THROW_MES(length >= TAG_SIZE, "Ciphertext length too small"); + CHECK_AND_ASSERT_THROW_MES(!plaintext_len || *plaintext_len >= (length - TAG_SIZE), "Plaintext length too small"); - unsigned long long int cip_len = length; + unsigned long long int res_len = plaintext_len ? *plaintext_len : length; auto r = crypto_aead_chacha20poly1305_ietf_decrypt( - reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr, + reinterpret_cast<unsigned char *>(plaintext), &res_len, nullptr, static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key); if (r != 0){ throw exc::Poly1305TagInvalid(); } + + if (plaintext_len){ + *plaintext_len = (size_t) res_len; + } } } @@ -185,6 +221,49 @@ namespace ki { } } + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + std::string str_out_key(out_key.data, sizeof(out_key.data)); + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, str_out_key, ack->salt()); + + const size_t len_ciphertext = ack->key_image().size(); // IV || keys + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t ki_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[ki_len]); + uint8_t * buff = plaintext.get(); + + protocol::crypto::chacha::decrypt( + ack->key_image().data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast<const uint8_t *>(enc_key.data), + reinterpret_cast<const uint8_t *>(ack->key_image().data()), + reinterpret_cast<char *>(buff), &ki_len); + + CHECK_AND_ASSERT_THROW_MES(ki_len == 3*32, "Invalid size"); + ::crypto::signature sig{}; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + in_ephemeral.pub = out_key; + in_ephemeral.sec = ::crypto::null_skey; + + // Verification + std::vector<const ::crypto::public_key*> pkeys; + pkeys.push_back(&out_key); + + CHECK_AND_ASSERT_THROW_MES(rct::scalarmultKey(rct::ki2rct(ki), rct::curveOrder()) == rct::identity(), + "Key image out of validity domain: key image " << epee::string_tools::pod_to_hex(ki)); + + CHECK_AND_ASSERT_THROW_MES(::crypto::check_ring_signature((const ::crypto::hash&)ki, ki, pkeys, &sig), + "Signature failed for key image " << epee::string_tools::pod_to_hex(ki) + << ", signature " + epee::string_tools::pod_to_hex(sig) + << ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } } // Cold transaction signing @@ -198,6 +277,8 @@ namespace tx { void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ dst->set_amount(src->amount); dst->set_is_subaddress(src->is_subaddress); + dst->set_is_integrated(src->is_integrated); + dst->set_original(src->original); translate_address(dst->mutable_addr(), &(src->addr)); } @@ -267,9 +348,29 @@ namespace tx { return std::string(buff, offset); } + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt) + { + uint8_t hash[32]; + KECCAK_CTX ctx; + ::crypto::secret_key res; + + keccak_init(&ctx); + keccak_update(&ctx, (const uint8_t *) private_view_key.data, sizeof(private_view_key.data)); + if (!aux.empty()){ + keccak_update(&ctx, (const uint8_t *) aux.data(), aux.size()); + } + keccak_finish(&ctx, hash); + keccak(hash, sizeof(hash), hash, sizeof(hash)); + + hmac_keccak_hash(hash, (const uint8_t *) salt.data(), salt.size(), hash, sizeof(hash)); + memcpy(res.data, hash, sizeof(hash)); + memwipe(hash, sizeof(hash)); + return res; + } + TData::TData() { - in_memory = false; rsig_type = 0; + bp_version = 0; cur_input_idx = 0; cur_output_idx = 0; cur_batch_idx = 0; @@ -283,6 +384,7 @@ namespace tx { m_tx_idx = tx_idx; m_ct.tx_data = cur_tx(); m_multisig = false; + m_client_version = 1; } void Signer::extract_payment_id(){ @@ -308,8 +410,8 @@ namespace tx { } } - static unsigned get_rsig_type(bool use_bulletproof, size_t num_outputs){ - if (!use_bulletproof){ + static unsigned get_rsig_type(const rct::RCTConfig &rct_config, size_t num_outputs){ + if (rct_config.range_proof_type == rct::RangeProofBorromean){ return rct::RangeProofBorromean; } else if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ return rct::RangeProofMultiOutputBulletproof; @@ -392,8 +494,10 @@ namespace tx { m_ct.tx.version = 2; m_ct.tx.unlock_time = tx.unlock_time; + m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1); tsx_data.set_version(1); + tsx_data.set_client_version(client_version()); tsx_data.set_unlock_time(tx.unlock_time); tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); @@ -402,8 +506,12 @@ namespace tx { // Rsig decision auto rsig_data = tsx_data.mutable_rsig_data(); - m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size()); + m_ct.rsig_type = get_rsig_type(tx.rct_config, tx.splitted_dsts.size()); rsig_data->set_rsig_type(m_ct.rsig_type); + if (tx.rct_config.range_proof_type != rct::RangeProofBorromean){ + m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1); + rsig_data->set_bp_version((uint32_t) m_ct.bp_version); + } generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); @@ -437,7 +545,6 @@ namespace tx { } void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){ - m_ct.in_memory = false; if (ack->has_rsig_data()){ m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data()); } @@ -505,10 +612,6 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ sort_ki(); - if (in_memory()){ - return nullptr; - } - auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); @@ -516,15 +619,10 @@ namespace tx { } void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){ - if (in_memory()){ - return; - } + } std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){ - if (in_memory()){ - return nullptr; - } CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); @@ -536,7 +634,8 @@ namespace tx { translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); - if (!in_memory()) { + + if (client_version() == 0) { CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); res->set_pseudo_out(m_ct.pseudo_outs[idx]); @@ -547,9 +646,7 @@ namespace tx { } void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){ - if (in_memory()){ - return; - } + } std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){ @@ -557,34 +654,37 @@ namespace tx { } void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ - if (is_offloading()){ - // If offloading, expect rsig configuration. - if (!ack->has_rsig_data()){ - throw exc::ProtocolException("Rsig offloading requires rsig param"); - } + if (client_version() > 0 || !is_offloading()){ + return; + } - auto & rsig_data = ack->rsig_data(); - if (!rsig_data.has_mask()){ - throw exc::ProtocolException("Gamma masks not present in offloaded version"); - } + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } - auto & mask = rsig_data.mask(); - if (mask.size() != 32 * num_outputs()){ - throw exc::ProtocolException("Invalid number of gamma masks"); - } + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } - m_ct.rsig_gamma.reserve(num_outputs()); - for(size_t c=0; c < num_outputs(); ++c){ - rct::key cmask{}; - memcpy(cmask.bytes, mask.data() + c * 32, 32); - m_ct.rsig_gamma.emplace_back(cmask); - } + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); } } std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported"); m_ct.cur_output_idx = idx; m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() @@ -595,48 +695,11 @@ namespace tx { res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); // Range sig offloading to the host - if (!is_offloading()) { - return res; - } - - CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); - if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) { - return res; - } - - auto rsig_data = res->mutable_rsig_data(); - auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; - - if (!is_req_bulletproof()){ - if (batch_size > 1){ - throw std::invalid_argument("Borromean cannot batch outputs"); - } - - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - rct::key C{}, mask = m_ct.rsig_gamma[idx]; - auto genRsig = rct::proveRange(C, mask, cur_dst.amount); // TODO: rsig with given mask - auto serRsig = cn_serialize(genRsig); - m_ct.tx_out_rsigs.emplace_back(genRsig); - rsig_data->set_rsig(serRsig); - - } else { - std::vector<uint64_t> amounts; - rct::keyV masks; - CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching"); - - for(size_t i = 0; i < batch_size; ++i){ - const size_t bidx = 1 + idx - batch_size + i; - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - - amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); - masks.push_back(m_ct.rsig_gamma[bidx]); - } - - auto bp = bulletproof_PROVE(amounts, masks); - auto serRsig = cn_serialize(bp); - m_ct.tx_out_rsigs.emplace_back(bp); - rsig_data->set_rsig(serRsig); + // ClientV0 sends offloaded BP with the last message in the batch. + // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks. + if (client_version() == 0 && is_offloading() && should_compute_bp_now()) { + auto rsig_data = res->mutable_rsig_data(); + compute_bproof(*rsig_data); } return res; @@ -644,7 +707,6 @@ namespace tx { void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ cryptonote::tx_out tx_out; - rct::rangeSig range_sig{}; rct::Bulletproof bproof{}; rct::ctkey out_pk{}; rct::ecdhTuple ecdh{}; @@ -658,12 +720,12 @@ namespace tx { if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ has_rsig = true; rsig_buff = rsig_data.rsig(); + } - } else if (rsig_data.rsig_parts_size() > 0){ - has_rsig = true; - for (const auto &it : rsig_data.rsig_parts()) { - rsig_buff += it; - } + if (client_version() >= 1 && rsig_data.has_mask()){ + rct::key cmask{}; + string_to_key(cmask, rsig_data.mask()); + m_ct.rsig_gamma.emplace_back(cmask); } } @@ -675,12 +737,13 @@ namespace tx { throw exc::ProtocolException("Cannot deserialize out_pk"); } - if (!cn_deserialize(ack->ecdh_info(), ecdh)){ - throw exc::ProtocolException("Cannot deserialize ecdhtuple"); - } - - if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){ - throw exc::ProtocolException("Cannot deserialize rangesig"); + if (m_ct.bp_version <= 1) { + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + } else { + CHECK_AND_ASSERT_THROW_MES(8 == ack->ecdh_info().size(), "Invalid ECDH.amount size"); + memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8); } if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ @@ -692,35 +755,77 @@ namespace tx { m_ct.tx_out_pk.emplace_back(out_pk); m_ct.tx_out_ecdh.emplace_back(ecdh); - if (!has_rsig){ + // ClientV0, if no rsig was generated on Trezor, do not continue. + // ClientV1+ generates BP after all masks in the current batch are generated + if (!has_rsig || (client_version() >= 1 && is_offloading())){ return; } - if (is_req_bulletproof()){ - CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); - auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; - for (size_t i = 0; i < batch_size; ++i){ - const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + process_bproof(bproof); + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } - rct::key commitment = m_ct.tx_out_pk[bidx].mask; - commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); - bproof.V.push_back(commitment); - } + bool Signer::should_compute_bp_now() const { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + return m_ct.grouping_vct[m_ct.cur_batch_idx] <= m_ct.cur_output_in_batch_idx; + } - m_ct.tx_out_rsigs.emplace_back(bproof); - if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); - } + void Signer::compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data){ + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + std::vector<uint64_t> amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_output_idx + 1 >= batch_size, "Invalid index for batching"); - } else { - m_ct.tx_out_rsigs.emplace_back(range_sig); + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); - } + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); } + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data.set_rsig(serRsig); + } + + void Signer::process_bproof(rct::Bulletproof & bproof){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){ + if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){ + return nullptr; + } + + auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + compute_bproof(*(res->mutable_rsig_data())); + res->set_is_offloaded_bp(true); + return res; + } + + void Signer::step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ m_ct.cur_batch_idx += 1; m_ct.cur_output_in_batch_idx = 0; } @@ -814,12 +919,11 @@ namespace tx { res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); res->set_pseudo_out_alpha(m_ct.alphas[idx]); res->set_spend_key(m_ct.spend_encs[idx]); - if (!in_memory()){ - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); - res->set_pseudo_out(m_ct.pseudo_outs[idx]); - res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); - } + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); return res; } @@ -829,6 +933,19 @@ namespace tx { throw exc::ProtocolException("Cannot deserialize mg[i]"); } + // Sync updated pseudo_outputs, client_version>=1, HF10+ + if (client_version() >= 1 && ack->has_pseudo_out()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index"); + m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out(); + if (is_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->p.pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->p.pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } else { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } + } + m_ct.rv->p.MGs.push_back(mg); } @@ -841,14 +958,14 @@ namespace tx { if (m_multisig){ auto & cout_key = ack->cout_key(); for(auto & cur : m_ct.couts){ - if (cur.size() != 12 + 32){ + if (cur.size() != crypto::chacha::IV_SIZE + 32){ throw std::invalid_argument("Encrypted cout has invalid length"); } char buff[32]; auto data = cur.data(); - crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); + crypto::chacha::decrypt(data + crypto::chacha::IV_SIZE, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); m_ct.couts_dec.emplace_back(buff, 32); } } @@ -887,6 +1004,82 @@ namespace tx { return sb.GetString(); } + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data) + { + rapidjson::Document json; + + // The contents should be JSON if the wallet follows the new format. + if (json.Parse(data.c_str()).HasParseError()) + { + throw std::invalid_argument("Data parsing error"); + } + else if(!json.IsObject()) + { + throw std::invalid_argument("Data parsing error - not an object"); + } + + GET_FIELD_FROM_JSON(json, version, int, Int, true, -1); + GET_STRING_FROM_JSON(json, salt1, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, salt2, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, enc_keys, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, tx_prefix_hash, std::string, false, std::string()); + + if (field_version != 1) + { + throw std::invalid_argument("Unknown version"); + } + + res.salt1 = field_salt1; + res.salt2 = field_salt2; + res.tx_enc_keys = field_enc_keys; + res.tx_prefix_hash = field_tx_prefix_hash; + } + + std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data) + { + auto req = std::make_shared<messages::monero::MoneroGetTxKeyRequest>(); + req->set_salt1(tx_data.salt1); + req->set_salt2(tx_data.salt2); + req->set_tx_enc_keys(tx_data.tx_enc_keys); + req->set_tx_prefix_hash(tx_data.tx_prefix_hash); + req->set_reason(0); + + return req; + } + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack + ) + { + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, tx_prefix_hash, ack->salt()); + auto & encrypted_keys = ack->has_tx_derivations() ? ack->tx_derivations() : ack->tx_keys(); + + const size_t len_ciphertext = encrypted_keys.size(); // IV || keys || TAG + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t keys_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[keys_len]); + + protocol::crypto::chacha::decrypt( + encrypted_keys.data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast<const uint8_t *>(enc_key.data), + reinterpret_cast<const uint8_t *>(encrypted_keys.data()), + reinterpret_cast<char *>(plaintext.get()), &keys_len); + + CHECK_AND_ASSERT_THROW_MES(keys_len % 32 == 0, "Invalid size"); + tx_keys.resize(keys_len / 32); + + for(unsigned i = 0; i < keys_len / 32; ++i) + { + memcpy(tx_keys[i].data, plaintext.get() + 32 * i, 32); + } + memwipe(plaintext.get(), keys_len); + } } } diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index 23f94f948..f58bf1039 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -92,11 +92,14 @@ namespace protocol{ // Crypto / encryption namespace crypto { namespace chacha { + // Constants as defined in RFC 7539. + const unsigned IV_SIZE = 12; + const unsigned TAG_SIZE = 16; // crypto_aead_chacha20poly1305_IETF_ABYTES; /** * Chacha20Poly1305 decryption with tag verification. RFC 7539. */ - void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext); + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len=nullptr); } } @@ -129,6 +132,14 @@ namespace ki { const std::vector<tools::wallet2::transfer_details> & transfers, std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + /** + * Processes Live refresh step response, parses KI, checks the signature + */ + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki); } // Cold transaction signing @@ -153,6 +164,7 @@ namespace tx { std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt); typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; @@ -164,8 +176,8 @@ namespace tx { TsxData tsx_data; tx_construction_data tx_data; cryptonote::transaction tx; - bool in_memory; unsigned rsig_type; + int bp_version; std::vector<uint64_t> grouping_vct; std::shared_ptr<MoneroRsigData> rsig_param; size_t cur_input_idx; @@ -206,6 +218,7 @@ namespace tx { const unsigned_tx_set * m_unsigned_tx; hw::tx_aux_data * m_aux_data; + unsigned m_client_version; bool m_multisig; const tx_construction_data & cur_tx(){ @@ -215,6 +228,9 @@ namespace tx { void extract_payment_id(); void compute_integrated_indices(TsxData * tsx_data); + bool should_compute_bp_now() const; + void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data); + void process_bproof(rct::Bulletproof & bproof); public: Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); @@ -238,6 +254,9 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx); void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_rsig(size_t idx); + void step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set(); void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev); @@ -249,8 +268,8 @@ namespace tx { std::string store_tx_aux_info(); - bool in_memory() const { - return m_ct.in_memory; + unsigned client_version() const { + return m_client_version; } bool is_simple() const { @@ -262,7 +281,7 @@ namespace tx { } bool is_req_bulletproof() const { - return m_ct.tx_data.use_bulletproofs; + return m_ct.tx_data.rct_config.range_proof_type != rct::RangeProofBorromean; } bool is_bulletproof() const { @@ -290,6 +309,18 @@ namespace tx { } }; + // TX Key decryption + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data); + + std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data); + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack + ); } } diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt new file mode 100644 index 000000000..1f369f114 --- /dev/null +++ b/src/lmdb/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(lmdb_sources database.cpp error.cpp table.cpp value_stream.cpp) +set(lmdb_headers database.h error.h key_stream.h table.h transaction.h util.h value_stream.h) + +monero_add_library(lmdb_lib ${lmdb_sources} ${lmdb_headers}) +target_link_libraries(lmdb_lib common ${LMDB_LIBRARY}) diff --git a/src/lmdb/database.cpp b/src/lmdb/database.cpp new file mode 100644 index 000000000..c6b244671 --- /dev/null +++ b/src/lmdb/database.cpp @@ -0,0 +1,187 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "database.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +#ifdef _WIN32 +namespace +{ + constexpr const mdb_mode_t open_flags = 0; +} +#else +#include <sys/stat.h> + +namespace +{ + constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR); +} +#endif + +namespace lmdb +{ + namespace + { + constexpr const std::size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB + void acquire_context(context& ctx) noexcept + { + while (ctx.lock.test_and_set()); + ++(ctx.active); + ctx.lock.clear(); + } + + void release_context(context& ctx) noexcept + { + --(ctx.active); + } + } + + void release_read_txn::operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + { + MDB_env* const env = mdb_txn_env(ptr); + abort_txn{}(ptr); + if (env) + { + context* ctx = reinterpret_cast<context*>(mdb_env_get_userctx(env)); + if (ctx) + release_context(*ctx); + } + } + } + + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept + { + MONERO_PRECOND(path != nullptr); + + MDB_env* obj = nullptr; + MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj))); + environment out{obj}; + + MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs)); + MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags)); + return {std::move(out)}; + } + + expect<write_txn> database::do_create_txn(unsigned int flags) noexcept + { + MONERO_PRECOND(handle() != nullptr); + + for (unsigned attempts = 0; attempts < 3; ++attempts) + { + acquire_context(ctx); + + MDB_txn* txn = nullptr; + const int err = + mdb_txn_begin(handle(), nullptr, flags, &txn); + if (!err && txn != nullptr) + return write_txn{txn}; + + release_context(ctx); + if (err != MDB_MAP_RESIZED) + return {lmdb::error(err)}; + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_RESIZED)}; + } + + database::database(environment env) + : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT} + { + if (handle()) + { + const int err = mdb_env_set_userctx(handle(), std::addressof(ctx)); + if (err) + MONERO_THROW(lmdb::error(err), "Failed to set user context"); + } + } + + database::~database() noexcept + { + while (ctx.active); + } + + expect<void> database::resize() noexcept + { + MONERO_PRECOND(handle() != nullptr); + + while (ctx.lock.test_and_set()); + while (ctx.active); + + MDB_envinfo info{}; + MONERO_LMDB_CHECK(mdb_env_info(handle(), &info)); + + const std::size_t resize = std::min(info.me_mapsize, max_resize); + const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize); + ctx.lock.clear(); + if (err) + return {lmdb::error(err)}; + return success(); + } + + expect<read_txn> database::create_read_txn(suspended_txn txn) noexcept + { + if (txn) + { + acquire_context(ctx); + const int err = mdb_txn_renew(txn.get()); + if (err) + { + release_context(ctx); + return {lmdb::error(err)}; + } + return read_txn{txn.release()}; + } + auto new_txn = do_create_txn(MDB_RDONLY); + if (new_txn) + return read_txn{new_txn->release()}; + return new_txn.error(); + } + + expect<suspended_txn> database::reset_txn(read_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + mdb_txn_reset(txn.get()); + release_context(ctx); + return suspended_txn{txn.release()}; + } + + expect<write_txn> database::create_write_txn() noexcept + { + return do_create_txn(0); + } + + expect<void> database::commit(write_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + MONERO_LMDB_CHECK(mdb_txn_commit(txn.get())); + txn.release(); + release_context(ctx); + return success(); + } +} // lmdb diff --git a/src/lmdb/database.h b/src/lmdb/database.h new file mode 100644 index 000000000..269f8c8a1 --- /dev/null +++ b/src/lmdb/database.h @@ -0,0 +1,138 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <atomic> +#include <cstddef> +#include <lmdb.h> +#include <memory> +#include <type_traits> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/transaction.h" + +namespace lmdb +{ + //! Closes LMDB environment handle. + struct close_env + { + void operator()(MDB_env* ptr) const noexcept + { + if (ptr) + mdb_env_close(ptr); + } + }; + + using environment = std::unique_ptr<MDB_env, close_env>; + + //! \return LMDB environment at `path` with a max of `max_dbs` tables. + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept; + + //! Context given to LMDB. + struct context + { + std::atomic<std::size_t> active; + std::atomic_flag lock; + }; + + //! Manages a LMDB environment for safe memory-map resizing. Thread-safe. + class database + { + environment env; + context ctx; + + //! \return The LMDB environment associated with the object. + MDB_env* handle() const noexcept { return env.get(); } + + expect<write_txn> do_create_txn(unsigned int flags) noexcept; + + public: + database(environment env); + + database(database&&) = delete; + database(database const&) = delete; + + virtual ~database() noexcept; + + database& operator=(database&&) = delete; + database& operator=(database const&) = delete; + + /*! + Resize the memory map for the LMDB environment. Will block until + all reads/writes on the environment complete. + */ + expect<void> resize() noexcept; + + //! \return A read only LMDB transaction, reusing `txn` if provided. + expect<read_txn> create_read_txn(suspended_txn txn = nullptr) noexcept; + + //! \return `txn` after releasing context. + expect<suspended_txn> reset_txn(read_txn txn) noexcept; + + //! \return A read-write LMDB transaction. + expect<write_txn> create_write_txn() noexcept; + + //! Commit the read-write transaction. + expect<void> commit(write_txn txn) noexcept; + + /*! + Create a write transaction, pass it to `f`, then try to commit + the write if `f` succeeds. + + \tparam F must be callable with signature `expect<T>(MDB_txn&)`. + \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`. + + \return The result of calling `f`. + */ + template<typename F> + typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3) + { + for (unsigned i = 0; i < attempts; ++i) + { + expect<write_txn> txn = create_write_txn(); + if (!txn) + return txn.error(); + + MONERO_PRECOND(*txn != nullptr); + const auto wrote = f(*(*txn)); + if (wrote) + { + MONERO_CHECK(commit(std::move(*txn))); + return wrote; + } + if (wrote != lmdb::error(MDB_MAP_FULL)) + return wrote; + + txn->reset(); + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_FULL)}; + } + }; +} // lmdb + diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp new file mode 100644 index 000000000..359677064 --- /dev/null +++ b/src/lmdb/error.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include <lmdb.h> +#include <string> + +namespace { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "lmdb::error_category()"; + } + + virtual std::string message(int value) const override final + { + char const* const msg = mdb_strerror(value); + if (msg) + return msg; + return "Unknown lmdb::error_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + switch (value) + { + case MDB_KEYEXIST: + case MDB_NOTFOUND: + break; // map to nothing generic + case MDB_PAGE_NOTFOUND: + case MDB_CORRUPTED: + return std::errc::state_not_recoverable; + case MDB_PANIC: + case MDB_VERSION_MISMATCH: + case MDB_INVALID: + break; // map to nothing generic + case MDB_MAP_FULL: + return std::errc::no_buffer_space; + case MDB_DBS_FULL: + break; // map to nothing generic + case MDB_READERS_FULL: + case MDB_TLS_FULL: + return std::errc::no_lock_available; + case MDB_TXN_FULL: + case MDB_CURSOR_FULL: + case MDB_PAGE_FULL: + case MDB_MAP_RESIZED: + break; // map to nothing generic + case MDB_INCOMPATIBLE: + return std::errc::invalid_argument; + case MDB_BAD_RSLOT: + case MDB_BAD_TXN: + case MDB_BAD_VALSIZE: + case MDB_BAD_DBI: + return std::errc::invalid_argument; + default: + return std::error_condition{value, std::generic_category()}; + } + return std::error_condition{value, *this}; + } + }; +} + +namespace lmdb +{ + std::error_category const& error_category() noexcept + { + static const category instance{}; + return instance; + } +} + diff --git a/src/lmdb/error.h b/src/lmdb/error.h new file mode 100644 index 000000000..2944adf78 --- /dev/null +++ b/src/lmdb/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <system_error> +#include <type_traits> + +//! Executes a LMDB command, and returns errors via `lmdb::error` enum. +#define MONERO_LMDB_CHECK(...) \ + do \ + { \ + const int err = __VA_ARGS__ ; \ + if (err) \ + return {lmdb::error(err)}; \ + } while (0) + +namespace lmdb +{ + //! Tracks LMDB error codes. + enum class error : int + { + // 0 is reserved for no error, as per expect<T> + // All other errors are the values reported by LMDB + }; + + std::error_category const& error_category() noexcept; + + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::lmdb::error> + : true_type + {}; +} diff --git a/src/lmdb/key_stream.h b/src/lmdb/key_stream.h new file mode 100644 index 000000000..40434d3a1 --- /dev/null +++ b/src/lmdb/key_stream.h @@ -0,0 +1,264 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "lmdb/value_stream.h" +#include "span.h" + +namespace lmdb +{ + + /*! + An InputIterator for a fixed-sized LMDB key and value. `operator++` + iterates over keys. + + \tparam K Key type in database records. + \tparam V Value type in database records. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename K, typename V> + class key_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> key; + + void increment() + { + // MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/ + if (cur) + key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first; + } + + public: + using value_type = std::pair<K, boost::iterator_range<value_iterator<V>>>; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + key_iterator() noexcept + : cur(nullptr), key() + {} + + /*! + \param cur Iterate over keys starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + key_iterator(MDB_cursor* cur) + : cur(cur), key() + { + if (cur) + key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first; + } + + //! \return True if `this` is one-past the last key. + bool is_end() const noexcept { return key.empty(); } + + //! \return True iff `rhs` is referencing `this` key. + bool equal(key_iterator const& rhs) const noexcept + { + return + (key.empty() && rhs.key.empty()) || + key.data() == rhs.key.data(); + } + + /*! + Moves iterator to next key or end. Invalidates all prior copies of + the iterator. + */ + key_iterator& operator++() + { + increment(); + return *this; + } + + /*! + Moves iterator to next key or end. + + \return A copy that is already invalidated, ignore + */ + key_iterator operator++(int) + { + key_iterator out{*this}; + increment(); + return out; + } + + //! \pre `!is_end()` \return {current key, current value range} + value_type operator*() const + { + return {get_key(), make_value_range()}; + } + + //! \pre `!is_end()` \return Current key + K get_key() const noexcept + { + assert(!is_end()); + K out; + std::memcpy(std::addressof(out), key.data(), sizeof(out)); + return out; + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return an entire value (`V`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + value_iterator<T, F, offset> make_value_iterator() const + { + static_assert(std::is_same<T, V>(), "bad MONERO_FIELD usage?"); + return {cur}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + boost::iterator_range<value_iterator<T, F, offset>> make_value_range() const + { + return {make_value_iterator<T, F, offset>(), value_iterator<T, F, offset>{}}; + } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and + value `V`. + + \tparam K key type being stored by each record. + \tparam V value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename K, typename V, typename D> + class key_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit key_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + key_stream(key_stream&&) = default; + key_stream(key_stream const&) = delete; + ~key_stream() = default; + key_stream& operator=(key_stream&&) = default; + key_stream& operator=(key_stream const&) = delete; + + /*! + Give up ownership of the cursor. `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first key/value. Newly created + iterators will start at the first value again. + + \note Invalidates all current iterators, including those created + with `make_iterator` or `make_range`. Also invalidates all + `value_iterator`s created with `key_iterator`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator over database keys from current cursor + position that will reach `.is_end()` after the last key. + */ + key_iterator<K, V> make_iterator() const + { + return {cur.get()}; + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Range from current cursor position until last key record. + Useful in for-each range loops or in templated code + */ + boost::iterator_range<key_iterator<K, V>> make_range() const + { + return {make_iterator(), key_iterator<K, V>{}}; + } + }; + + template<typename K, typename V> + inline + bool operator==(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename K, typename V> + inline + bool operator!=(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/lmdb/table.cpp b/src/lmdb/table.cpp new file mode 100644 index 000000000..0818b74e6 --- /dev/null +++ b/src/lmdb/table.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "table.h" + +namespace lmdb +{ + expect<MDB_dbi> table::open(MDB_txn& write_txn) const noexcept + { + MONERO_PRECOND(name != nullptr); + + MDB_dbi out; + MONERO_LMDB_CHECK(mdb_dbi_open(&write_txn, name, flags, &out)); + if (key_cmp && !(flags & MDB_INTEGERKEY)) + MONERO_LMDB_CHECK(mdb_set_compare(&write_txn, out, key_cmp)); + if (value_cmp && !(flags & MDB_INTEGERDUP)) + MONERO_LMDB_CHECK(mdb_set_dupsort(&write_txn, out, value_cmp)); + return out; + } +} diff --git a/src/lmdb/table.h b/src/lmdb/table.h new file mode 100644 index 000000000..41a3de296 --- /dev/null +++ b/src/lmdb/table.h @@ -0,0 +1,120 @@ +#pragma once + +#include <utility> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/key_stream.h" +#include "lmdb/util.h" +#include "lmdb/value_stream.h" + +namespace lmdb +{ + //! Helper for grouping typical LMDB DBI options. + struct table + { + char const* const name; + const unsigned flags; + MDB_cmp_func const* const key_cmp; + MDB_cmp_func const* const value_cmp; + + //! \pre `name != nullptr` \return Open table. + expect<MDB_dbi> open(MDB_txn& write_txn) const noexcept; + }; + + //! Helper for grouping typical LMDB DBI options when key and value are fixed types. + template<typename K, typename V> + struct basic_table : table + { + using key_type = K; + using value_type = V; + + //! \return Additional LMDB flags based on `flags` value. + static constexpr unsigned compute_flags(const unsigned flags) noexcept + { + return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0); + } + + constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : table{name, compute_flags(flags), &lmdb::less<lmdb::native_type<K>>, value_cmp} + {} + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template<typename U, typename F = U, std::size_t offset = 0> + static expect<F> get_value(MDB_val value) noexcept + { + static_assert(std::is_same<U, V>(), "bad MONERO_FIELD?"); + static_assert(std::is_pod<F>(), "F must be POD"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size != sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast<char*>(value.mv_data) + offset, sizeof(out)); + return out; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first key/value in the table linked + to `cur` or an empty `key_stream`. + */ + template<typename D> + expect<key_stream<K, V, D>> + static get_key_stream(std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key; + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return key_stream<K, V, D>{std::move(cur)}; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first value at `key` in the table linked + to `cur` or an empty `value_stream`. + */ + template<typename D> + expect<value_stream<V, D>> + static get_value_stream(K const& key, std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return value_stream<V, D>{std::move(cur)}; + } + }; +} // lmdb + diff --git a/src/lmdb/transaction.h b/src/lmdb/transaction.h new file mode 100644 index 000000000..cdd80696c --- /dev/null +++ b/src/lmdb/transaction.h @@ -0,0 +1,95 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <lmdb.h> +#include <memory> + +#include "lmdb/error.h" + +//! Uses C++ type system to differentiate between cursors +#define MONERO_CURSOR(name) \ + struct close_ ## name : ::lmdb::close_cursor {}; \ + using name = std::unique_ptr< MDB_cursor, close_ ## name >; + +namespace lmdb +{ + struct abort_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + mdb_txn_abort(ptr); + } + }; + + /*! + Only valid if used via `create_read_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct release_read_txn + { + void operator()(MDB_txn* ptr) const noexcept; + // implementation in database.cpp + }; + + /*! + Only valid if used via `create_write_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct abort_write_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + release_read_txn{}(ptr); + } + }; + + struct close_cursor + { + void operator()(MDB_cursor* ptr) const noexcept + { + if (ptr) + mdb_cursor_close(ptr); + } + }; + + template<typename D> + inline expect<std::unique_ptr<MDB_cursor, D>> + open_cursor(MDB_txn& txn, MDB_dbi tbl) noexcept + { + MDB_cursor* cur = nullptr; + MONERO_LMDB_CHECK(mdb_cursor_open(&txn, tbl, &cur)); + return std::unique_ptr<MDB_cursor, D>{cur}; + } + + // The below use the C++ type system to designate `MDB_txn` status. + + using suspended_txn = std::unique_ptr<MDB_txn, abort_txn>; + using read_txn = std::unique_ptr<MDB_txn, release_read_txn>; + using write_txn = std::unique_ptr<MDB_txn, abort_write_txn>; +} // lmdb diff --git a/src/lmdb/util.h b/src/lmdb/util.h new file mode 100644 index 000000000..50162b7c8 --- /dev/null +++ b/src/lmdb/util.h @@ -0,0 +1,149 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <cstddef> +#include <cstring> +#include <lmdb.h> +#include <type_traits> +#include <utility> + +#include "span.h" + +/*! Calculates types and offset of struct field. Use in template arguments for + `table::get_value`, `value_iterator::get_value`, + `value_stream::make_iterator`, or `value_stream::make_range`. */ +#define MONERO_FIELD(obj, field) \ + obj , decltype(std::declval<obj>().field) , offsetof(obj, field) + +//! Expands to `lmdb::less` for the value `field` within `obj`. +#define MONERO_SORT_BY(obj, field) \ + &::lmdb::less< \ + lmdb::native_type<decltype(std::declval<obj>().field)>, \ + offsetof(obj, field) \ + > + +//! Expands to `lmdb::compare` for the value `field` within `obj`. +#define MONERO_COMPARE(obj, field) \ + &::lmdb::compare< \ + decltype(std::declval<obj>().field), \ + offsetof(obj, field) \ + > + +namespace lmdb +{ + //! Prevent instantiation of `std::underlying_type<T>` when `T` is not enum. + template<typename T> + struct identity + { + using type = T; + }; + + /*! + Get the native type for enums, or return `T` unchanged. Useful for + merging generated machine code for templated functions that use enums + with identical size-widths without relying on aggressive identical + comdat folding (ICF) support in linker. So with enum defintion + `enum class enum_foo : unsigned long {};` will always yield + `assert(&func_foo<unsigned long> == &func_foo<native_type<enum_foo>>)`. + */ + template<typename T> + using native_type = typename std::conditional< + std::is_enum<T>::value, std::underlying_type<T>, identity<T> + >::type::type; + + //! \return `value` as its native type. + template<typename T, typename U = typename std::underlying_type<T>::type> + inline constexpr U to_native(T value) noexcept + { + return U(value); + } + + //! \return `value` bytes in a LMDB `MDB_val` object. + template<typename T> + inline MDB_val to_val(T&& value) noexcept + { + // lmdb does not touch user data, so const_cast is acceptable + static_assert(!std::is_rvalue_reference<T&&>(), "cannot use temporary value"); + void const* const temp = reinterpret_cast<void const*>(std::addressof(value)); + return MDB_val{sizeof(value), const_cast<void*>(temp)}; + } + + //! \return A span over the same chunk of memory as `value`. + inline constexpr epee::span<const std::uint8_t> to_byte_span(MDB_val value) noexcept + { + return {static_cast<const std::uint8_t*>(value.mv_data), value.mv_size}; + } + + /*! + A LMDB comparison function that uses `operator<`. + + \tparam T has a defined `operator<` . + \tparam offset to `T` within the value. + + \return -1 if `left < right`, 1 if `right < left`, and 0 otherwise. + */ + template<typename T, std::size_t offset = 0> + inline int less(MDB_val const* left, MDB_val const* right) noexcept + { + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + + T left_val; + T right_val; + std::memcpy(std::addressof(left_val), static_cast<char*>(left->mv_data) + offset, sizeof(T)); + std::memcpy(std::addressof(right_val), static_cast<char*>(right->mv_data) + offset, sizeof(T)); + return left_val < right_val ? -1 : bool(right_val < left_val); + } + + /*! + A LMDB comparison function that uses `std::memcmp`. + + \toaram T is `!epee::has_padding` + \tparam offset to `T` within the value. + + \return The result of `std::memcmp` over the value. + */ + template<typename T, std::size_t offset = 0> + inline int compare(MDB_val const* left, MDB_val const* right) noexcept + { + static_assert(!epee::has_padding<T>(), "memcmp will not work"); + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + return std::memcmp( + static_cast<char*>(left->mv_data) + offset, + static_cast<char*>(right->mv_data) + offset, + sizeof(T) + ); + } +} // lmdb diff --git a/src/lmdb/value_stream.cpp b/src/lmdb/value_stream.cpp new file mode 100644 index 000000000..1024deb06 --- /dev/null +++ b/src/lmdb/value_stream.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "value_stream.h" + +#include <stdexcept> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +namespace lmdb +{ + namespace stream + { + std::size_t count(MDB_cursor* cur) + { + std::size_t out = 0; + if (cur) + { + const int rc = mdb_cursor_count(cur, &out); + if (rc) + MONERO_THROW(lmdb::error(rc), "mdb_cursor_count"); + } + return out; + } + + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value) + { + MDB_val key_bytes{}; + MDB_val value_bytes{}; + const int rc = mdb_cursor_get(&cur, &key_bytes, &value_bytes, op); + if (rc) + { + if (rc == MDB_NOTFOUND) + return {}; + MONERO_THROW(lmdb::error(rc), "mdb_cursor_get"); + } + + if (key && key != key_bytes.mv_size) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get key"); + + if (value && (value_bytes.mv_size % value != 0 || value_bytes.mv_size == 0)) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get value"); + + return {lmdb::to_byte_span(key_bytes), lmdb::to_byte_span(value_bytes)}; + } + } +} + diff --git a/src/lmdb/value_stream.h b/src/lmdb/value_stream.h new file mode 100644 index 000000000..c9977221f --- /dev/null +++ b/src/lmdb/value_stream.h @@ -0,0 +1,287 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "span.h" + +namespace lmdb +{ + namespace stream + { + /* + \throw std::system_error if unexpected LMDB error. + \return 0 if `cur == nullptr`, otherwise count of values at current key. + */ + std::size_t count(MDB_cursor* cur); + + /*! + Calls `mdb_cursor_get` and does some error checking. + + \param cur is given to `mdb_cursor_get` without modification. + \param op is passed to `mdb_cursor_get` without modification. + \param key expected key size or 0 to skip key size check. + \param value expected value size or 0 to skip value size check. + + \throw std::system_error if `key != 0` and `key_.mv_size != key`. + \throw std::system_error if `value != 0` and `value_.mv_size != value`. + \throw std::system_error if `mdb_cursor_get` returns any error + other than `MDB_NOTFOUND`. + + \return {key bytes, value bytes} or two empty spans if `MDB_NOTFOUND`. + */ + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value); + } + + /*! + An InputIterator for a fixed-sized LMDB value at a specific key. + + \tparam T The value type at the specific key. + \tparam F The value type being returned when dereferenced. + \tparam offset to `F` within `T`. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename T, typename F = T, std::size_t offset = 0> + class value_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> values; + + void increment() + { + values.remove_prefix(sizeof(T)); + if (values.empty() && cur) + values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second; + } + + public: + using value_type = F; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + value_iterator() noexcept + : cur(nullptr), values() + {} + + /*! + \param cur Iterate over values starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + value_iterator(MDB_cursor* cur) + : cur(cur), values() + { + if (cur) + values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second; + } + + value_iterator(value_iterator const&) = default; + ~value_iterator() = default; + value_iterator& operator=(value_iterator const&) = default; + + //! \return True if `this` is one-past the last value. + bool is_end() const noexcept { return values.empty(); } + + //! \return True iff `rhs` is referencing `this` value. + bool equal(value_iterator const& rhs) const noexcept + { + return + (values.empty() && rhs.values.empty()) || + values.data() == rhs.values.data(); + } + + //! Invalidates all prior copies of the iterator. + value_iterator& operator++() + { + increment(); + return *this; + } + + //! \return A copy that is already invalidated, ignore + value_iterator operator++(int) + { + value_iterator out{*this}; + increment(); + return out; + } + + /*! + Get a specific field within `F`. Default behavior is to return + the entirety of `U`, despite the filtering logic of `operator*`. + + \pre `!is_end()` + + \tparam U must match `T`, used for `MONERO_FIELD` sanity checking. + \tparam G field type to extract from the value + \tparam uoffset to `G` type, or `0` when `std::is_same<U, G>()`. + + \return The field `G`, at `uoffset` within `U`. + */ + template<typename U, typename G = U, std::size_t uoffset = 0> + G get_value() const noexcept + { + static_assert(std::is_same<U, T>(), "bad MONERO_FIELD usage?"); + static_assert(std::is_pod<U>(), "value type must be pod"); + static_assert(std::is_pod<G>(), "field type must be pod"); + static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset"); + assert(sizeof(G) + uoffset <= values.size()); + assert(!is_end()); + + G value; + std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value)); + return value; + } + + //! \pre `!is_end()` \return The field `F`, at `offset`, within `T`. + value_type operator*() const noexcept { return get_value<T, F, offset>(); } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`. + + \tparam T value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename T, typename D> + class value_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit value_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + value_stream(value_stream&&) = default; + value_stream(value_stream const&) = delete; + ~value_stream() = default; + value_stream& operator=(value_stream&&) = default; + value_stream& operator=(value_stream const&) = delete; + + /*! + Give up ownership of the cursor. `count()`, `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first value. Newly created iterators + will start at the first value again. + + \note Invalidates all current iterators from `this`, including + those created with `make_iterator` or `make_range`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Number of values at this key. + */ + std::size_t count() const + { + return lmdb::stream::count(cur.get()); + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return entire value (`T`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + value_iterator<U, F, offset> make_iterator() const + { + static_assert(std::is_same<U, T>(), "was MONERO_FIELD used with wrong type?"); + return {cur.get()}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + boost::iterator_range<value_iterator<U, F, offset>> make_range() const + { + return {make_iterator<U, F, offset>(), value_iterator<U, F, offset>{}}; + } + }; + + template<typename T, typename F, std::size_t offset> + inline + bool operator==(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename T, typename F, std::size_t offset> + inline + bool operator!=(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 48c9ab1ba..2dd40cc9a 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -340,9 +340,7 @@ namespace crypto const size_t expected = len * 3 / 32; if (seed.size() == expected/2) { - dst += ' '; // if electrum 12-word seed, duplicate - dst += dst; // if electrum 12-word seed, duplicate - dst.pop_back(); // trailing space + dst.append(dst.data(), dst.size()); // if electrum 12-word seed, duplicate } } diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 8a3ee9e6f..738f858f0 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,8 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources error.cpp parse.cpp socks.cpp tor_address.cpp i2p_address.cpp) -set(net_headers error.h parse.h socks.h tor_address.h i2p_address.h) +set(net_sources error.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp) +set(net_headers error.h i2p_address.h parse.h socks.h socks_connect.h tor_address.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${Boost_ASIO_LIBRARY}) diff --git a/src/net/socks.cpp b/src/net/socks.cpp index 53154369b..5a27e16f4 100644 --- a/src/net/socks.cpp +++ b/src/net/socks.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Monero Project +// Copyright (c) 2018-2019, The Monero Project // // All rights reserved. // @@ -193,7 +193,7 @@ namespace socks else if (bytes < self.buffer().size()) self.done(socks::error::bad_write, std::move(self_)); else - boost::asio::async_read(self.proxy_, get_buffer(self), completed{std::move(self_)}); + boost::asio::async_read(self.proxy_, get_buffer(self), self.strand_.wrap(completed{std::move(self_)})); } } }; @@ -215,13 +215,13 @@ namespace socks if (error) self.done(error, std::move(self_)); else - boost::asio::async_write(self.proxy_, get_buffer(self), read{std::move(self_)}); + boost::asio::async_write(self.proxy_, get_buffer(self), self.strand_.wrap(read{std::move(self_)})); } } }; client::client(stream_type::socket&& proxy, socks::version ver) - : proxy_(std::move(proxy)), buffer_size_(0), buffer_(), ver_(ver) + : proxy_(std::move(proxy)), strand_(GET_IO_SERVICE(proxy_)), buffer_size_(0), buffer_(), ver_(ver) {} client::~client() {} @@ -296,7 +296,7 @@ namespace socks if (self && !self->buffer().empty()) { client& alias = *self; - alias.proxy_.async_connect(proxy_address, write{std::move(self)}); + alias.proxy_.async_connect(proxy_address, alias.strand_.wrap(write{std::move(self)})); return true; } return false; @@ -307,10 +307,26 @@ namespace socks if (self && !self->buffer().empty()) { client& alias = *self; - boost::asio::async_write(alias.proxy_, write::get_buffer(alias), read{std::move(self)}); + boost::asio::async_write(alias.proxy_, write::get_buffer(alias), alias.strand_.wrap(read{std::move(self)})); return true; } return false; } + + void client::async_close::operator()(boost::system::error_code error) + { + if (self_ && error != boost::system::errc::operation_canceled) + { + const std::shared_ptr<client> self = std::move(self_); + self->strand_.dispatch([self] () + { + if (self && self->proxy_.is_open()) + { + self->proxy_.shutdown(boost::asio::ip::tcp::socket::shutdown_both); + self->proxy_.close(); + } + }); + } + } } // socks } // net diff --git a/src/net/socks.h b/src/net/socks.h index 825937792..4d1d34e9e 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Monero Project +// Copyright (c) 2018-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #include <cstdint> #include <boost/asio/ip/tcp.hpp> #include <boost/asio/io_service.hpp> +#include <boost/asio/strand.hpp> #include <boost/system/error_code.hpp> #include <boost/type_traits/integral_constant.hpp> #include <boost/utility/string_ref.hpp> @@ -92,6 +93,7 @@ namespace socks class client { boost::asio::ip::tcp::socket proxy_; + boost::asio::io_service::strand strand_; std::uint16_t buffer_size_; std::uint8_t buffer_[1024]; socks::version ver_; @@ -168,6 +170,8 @@ namespace socks \note Must use one of the `self->set_*_command` calls before using this function. + \note Only `async_close` can be invoked on `self` until the `done` + callback is invoked. \param self ownership of object is given to function. \param proxy_address of the socks server. @@ -182,11 +186,21 @@ namespace socks \note Must use one of the `self->set_*_command` calls before using the function. + \note Only `async_close` can be invoked on `self` until the `done` + callback is invoked. \param self ownership of object is given to function. \return False if `self->buffer().empty()` (no command set). */ static bool send(std::shared_ptr<client> self); + + /*! Callback for closing socket. Thread-safe with `*send` functions; + never blocks (uses strands). */ + struct async_close + { + std::shared_ptr<client> self_; + void operator()(boost::system::error_code error = boost::system::error_code{}); + }; }; template<typename Handler> diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp new file mode 100644 index 000000000..a5557f6f8 --- /dev/null +++ b/src/net/socks_connect.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "socks_connect.h" + +#include <boost/system/error_code.hpp> +#include <boost/system/system_error.hpp> +#include <cstdint> +#include <memory> +#include <system_error> + +#include "net/error.h" +#include "net/net_utils_base.h" +#include "net/socks.h" +#include "string_tools.h" + +namespace net +{ +namespace socks +{ + boost::unique_future<boost::asio::ip::tcp::socket> + connector::operator()(const std::string& remote_host, const std::string& remote_port, boost::asio::steady_timer& timeout) const + { + struct future_socket + { + boost::promise<boost::asio::ip::tcp::socket> result_; + + void operator()(boost::system::error_code error, boost::asio::ip::tcp::socket&& socket) + { + if (error) + result_.set_exception(boost::system::system_error{error}); + else + result_.set_value(std::move(socket)); + } + }; + + boost::unique_future<boost::asio::ip::tcp::socket> out{}; + { + std::uint16_t port = 0; + if (!epee::string_tools::get_xtype_from_string(port, remote_port)) + throw std::system_error{net::error::invalid_port, "Remote port for socks proxy"}; + + bool is_set = false; + std::uint32_t ip_address = 0; + boost::promise<boost::asio::ip::tcp::socket> result{}; + out = result.get_future(); + const auto proxy = net::socks::make_connect_client( + boost::asio::ip::tcp::socket{GET_IO_SERVICE(timeout)}, net::socks::version::v4a, future_socket{std::move(result)} + ); + + if (epee::string_tools::get_ip_int32_from_string(ip_address, remote_host)) + is_set = proxy->set_connect_command(epee::net_utils::ipv4_network_address{ip_address, port}); + else + is_set = proxy->set_connect_command(remote_host, port); + + if (!is_set || !net::socks::client::connect_and_send(proxy, proxy_address)) + throw std::system_error{net::error::invalid_host, "Address for socks proxy"}; + + timeout.async_wait(net::socks::client::async_close{std::move(proxy)}); + } + + return out; + } +} // socks +} // net diff --git a/src/net/socks_connect.h b/src/net/socks_connect.h new file mode 100644 index 000000000..44b0fa2b3 --- /dev/null +++ b/src/net/socks_connect.h @@ -0,0 +1,55 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/asio/ip/tcp.hpp> +#include <boost/asio/steady_timer.hpp> +#include <boost/thread/future.hpp> +#include <string> + +namespace net +{ +namespace socks +{ + //! Primarily for use with `epee::net_utils::http_client`. + struct connector + { + boost::asio::ip::tcp::endpoint proxy_address; + + /*! Creates a new socket, asynchronously connects to `proxy_address`, + and requests a connection to `remote_host` on `remote_port`. Sets + socket as closed if `timeout` is reached. + + \return The socket if successful, and exception in the future with + error otherwise. */ + boost::unique_future<boost::asio::ip::tcp::socket> + operator()(const std::string& remote_host, const std::string& remote_port, boost::asio::steady_timer& timeout) const; + }; +} // socks +} // net diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index e39ba16fd..b5499262f 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -408,10 +408,10 @@ namespace rct { return res; } - //Computes aL where L is the curve order - bool isInMainSubgroup(const key & a) { + //Computes lA where l is the curve order + bool isInMainSubgroup(const key & A) { ge_p3 p3; - return toPointCheckOrder(&p3, a.bytes); + return toPointCheckOrder(&p3, A.bytes); } //Curve addition / subtractions diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index ff55167fe..e6a09606b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -70,6 +70,13 @@ namespace { return (value + quantum - 1) / quantum * quantum; } + + void store_difficulty(cryptonote::difficulty_type difficulty, uint64_t &sdiff, std::string &swdiff, uint64_t &stop64) + { + sdiff = (difficulty << 64 >> 64).convert_to<uint64_t>(); + swdiff = difficulty.convert_to<std::string>(); + stop64 = (difficulty >> 64).convert_to<uint64_t>(); + } } namespace cryptonote @@ -219,7 +226,7 @@ namespace cryptonote ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); res.target_height = m_core.get_target_blockchain_height(); - res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block(); + store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); @@ -236,7 +243,8 @@ namespace cryptonote res.testnet = net_type == TESTNET; res.stagenet = net_type == STAGENET; res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; - res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1), + res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; @@ -1196,13 +1204,15 @@ namespace cryptonote block b; cryptonote::blobdata blob_reserve; blob_reserve.resize(req.reserve_size, 0); - if(!m_core.get_block_template(b, info.address, res.difficulty, res.height, res.expected_reward, blob_reserve)) + cryptonote::difficulty_type wdiff; + if(!m_core.get_block_template(b, info.address, wdiff, res.height, res.expected_reward, blob_reserve)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; LOG_ERROR("Failed to create block template"); return false; } + store_difficulty(wdiff, res.difficulty, res.wide_difficulty, res.difficulty_top64); blobdata block_blob = t_serializable_object_to_blob(b); crypto::public_key tx_pub_key = cryptonote::get_tx_pub_key_from_extra(b.miner_tx); if(tx_pub_key == crypto::null_pkey) @@ -1375,8 +1385,10 @@ namespace cryptonote response.height = height; response.depth = m_core.get_current_blockchain_height() - height - 1; response.hash = string_tools::pod_to_hex(hash); - response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); - response.cumulative_difficulty = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height); + store_difficulty(m_core.get_blockchain_storage().block_difficulty(height), + response.difficulty, response.wide_difficulty, response.difficulty_top64); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height), + response.cumulative_difficulty, response.wide_cumulative_difficulty, response.cumulative_difficulty_top64); response.reward = get_block_reward(blk); response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); @@ -1471,7 +1483,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get last block."; return false; } - bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1513,7 +1526,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1563,7 +1577,8 @@ namespace cryptonote return false; } res.headers.push_back(block_header_response()); - bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1597,7 +1612,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get block by height. Height = " + std::to_string(req.height) + '.'; return false; } - bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1652,7 +1668,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1702,7 +1719,8 @@ namespace cryptonote ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); res.target_height = m_core.get_target_blockchain_height(); - res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block(); + store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), + res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); @@ -1720,7 +1738,8 @@ namespace cryptonote res.stagenet = net_type == STAGENET; res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; - res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1), + res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; @@ -1736,7 +1755,9 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } - res.database_size = restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); + if (restricted) + res.database_size = round_up(res.database_size, 5ull * 1024 * 1024 * 1024); res.update_available = restricted ? false : m_core.is_update_available(); res.version = restricted ? "" : MONERO_VERSION; return true; @@ -1942,7 +1963,9 @@ namespace cryptonote std::list<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains(); for (const auto &i: chains) { - res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), i.first.cumulative_difficulty, {}, std::string()}); + difficulty_type wdiff = i.first.cumulative_difficulty; + res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), 0, "", 0, {}, std::string()}); + store_difficulty(wdiff, res.chains.back().difficulty, res.chains.back().wide_difficulty, res.chains.back().difficulty_top64); res.chains.back().block_hashes.reserve(i.second.size()); for (const crypto::hash &block_id: i.second) res.chains.back().block_hashes.push_back(epee::string_tools::pod_to_hex(block_id)); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index f65c7c8dd..4786f7a9b 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -252,223 +252,6 @@ namespace cryptonote }; //----------------------------------------------- - struct COMMAND_RPC_GET_ADDRESS_TXS - { - struct request_t - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct spent_output { - uint64_t amount; - std::string key_image; - std::string tx_pub_key; - uint64_t out_index; - uint32_t mixin; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(key_image) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(out_index) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - struct transaction - { - uint64_t id; - std::string hash; - uint64_t timestamp; - uint64_t total_received; - uint64_t total_sent; - uint64_t unlock_time; - uint64_t height; - std::list<spent_output> spent_outputs; - std::string payment_id; - bool coinbase; - bool mempool; - uint32_t mixin; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(id) - KV_SERIALIZE(hash) - KV_SERIALIZE(timestamp) - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_sent) - KV_SERIALIZE(unlock_time) - KV_SERIALIZE(height) - KV_SERIALIZE(spent_outputs) - KV_SERIALIZE(payment_id) - KV_SERIALIZE(coinbase) - KV_SERIALIZE(mempool) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - - struct response_t - { - //std::list<std::string> txs_as_json; - uint64_t total_received; - uint64_t total_received_unlocked = 0; // OpenMonero only - uint64_t scanned_height; - std::vector<transaction> transactions; - uint64_t blockchain_height; - uint64_t scanned_block_height; - std::string status; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_received_unlocked) - KV_SERIALIZE(scanned_height) - KV_SERIALIZE(transactions) - KV_SERIALIZE(blockchain_height) - KV_SERIALIZE(scanned_block_height) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_ADDRESS_INFO - { - struct request_t - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct spent_output - { - uint64_t amount; - std::string key_image; - std::string tx_pub_key; - uint64_t out_index; - uint32_t mixin; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(key_image) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(out_index) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - struct response_t - { - uint64_t locked_funds; - uint64_t total_received; - uint64_t total_sent; - uint64_t scanned_height; - uint64_t scanned_block_height; - uint64_t start_height; - uint64_t transaction_height; - uint64_t blockchain_height; - std::list<spent_output> spent_outputs; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(locked_funds) - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_sent) - KV_SERIALIZE(scanned_height) - KV_SERIALIZE(scanned_block_height) - KV_SERIALIZE(start_height) - KV_SERIALIZE(transaction_height) - KV_SERIALIZE(blockchain_height) - KV_SERIALIZE(spent_outputs) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_UNSPENT_OUTS - { - struct request_t - { - std::string amount; - std::string address; - std::string view_key; - // OpenMonero specific - uint64_t mixin; - bool use_dust; - std::string dust_threshold; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - KV_SERIALIZE(mixin) - KV_SERIALIZE(use_dust) - KV_SERIALIZE(dust_threshold) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - - struct output { - uint64_t amount; - std::string public_key; - uint64_t index; - uint64_t global_index; - std::string rct; - std::string tx_hash; - std::string tx_pub_key; - std::string tx_prefix_hash; - std::vector<std::string> spend_key_images; - uint64_t timestamp; - uint64_t height; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(public_key) - KV_SERIALIZE(index) - KV_SERIALIZE(global_index) - KV_SERIALIZE(rct) - KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(tx_prefix_hash) - KV_SERIALIZE(spend_key_images) - KV_SERIALIZE(timestamp) - KV_SERIALIZE(height) - END_KV_SERIALIZE_MAP() - }; - - struct response_t - { - uint64_t amount; - std::list<output> outputs; - uint64_t per_kb_fee; - std::string status; - std::string reason; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(outputs) - KV_SERIALIZE(per_kb_fee) - KV_SERIALIZE(status) - KV_SERIALIZE(reason) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - - //----------------------------------------------- struct COMMAND_RPC_GET_RANDOM_OUTS { struct request_t @@ -548,72 +331,6 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- - struct COMMAND_RPC_LOGIN - { - struct request_t - { - std::string address; - std::string view_key; - bool create_account; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - KV_SERIALIZE(create_account) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct response_t - { - std::string status; - std::string reason; - bool new_address; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(status) - KV_SERIALIZE(reason) - KV_SERIALIZE(new_address) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - //----------------------------------------------- - struct COMMAND_RPC_IMPORT_WALLET_REQUEST - { - struct request_t - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct response_t - { - std::string payment_id; - uint64_t import_fee; - bool new_request; - bool request_fulfilled; - std::string payment_address; - std::string status; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(payment_id) - KV_SERIALIZE(import_fee) - KV_SERIALIZE(new_request) - KV_SERIALIZE(request_fulfilled) - KV_SERIALIZE(payment_address) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { struct request_t @@ -943,6 +660,8 @@ namespace cryptonote uint64_t height; uint64_t target_height; uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; uint64_t target; uint64_t tx_count; uint64_t tx_pool_size; @@ -958,6 +677,8 @@ namespace cryptonote std::string nettype; std::string top_block_hash; uint64_t cumulative_difficulty; + std::string wide_cumulative_difficulty; + uint64_t cumulative_difficulty_top64; uint64_t block_size_limit; uint64_t block_weight_limit; uint64_t block_size_median; @@ -978,6 +699,8 @@ namespace cryptonote KV_SERIALIZE(height) KV_SERIALIZE(target_height) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(target) KV_SERIALIZE(tx_count) KV_SERIALIZE(tx_pool_size) @@ -993,6 +716,8 @@ namespace cryptonote KV_SERIALIZE(nettype) KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(wide_cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE(block_size_limit) KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) @@ -1149,6 +874,8 @@ namespace cryptonote struct response_t { uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; uint64_t height; uint64_t reserved_offset; uint64_t expected_reward; @@ -1160,6 +887,8 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(height) KV_SERIALIZE(reserved_offset) KV_SERIALIZE(expected_reward) @@ -1226,8 +955,12 @@ namespace cryptonote uint64_t height; uint64_t depth; std::string hash; - difficulty_type difficulty; - difficulty_type cumulative_difficulty; + uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; + uint64_t cumulative_difficulty; + std::string wide_cumulative_difficulty; + uint64_t cumulative_difficulty_top64; uint64_t reward; uint64_t block_size; uint64_t block_weight; @@ -1246,7 +979,11 @@ namespace cryptonote KV_SERIALIZE(depth) KV_SERIALIZE(hash) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(wide_cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) KV_SERIALIZE_OPT(block_weight, (uint64_t)0) @@ -2248,6 +1985,8 @@ namespace cryptonote uint64_t height; uint64_t length; uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; std::vector<std::string> block_hashes; std::string main_chain_parent_block; @@ -2256,6 +1995,8 @@ namespace cryptonote KV_SERIALIZE(height) KV_SERIALIZE(length) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(block_hashes) KV_SERIALIZE(main_chain_parent_block) END_KV_SERIALIZE_MAP() diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 14b492786..540afe6b9 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -263,20 +263,43 @@ namespace rpc void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { - auto tx_blob = cryptonote::tx_to_blob(req.tx); + handleTxBlob(cryptonote::tx_to_blob(req.tx), req.relay, res); + } + + void DaemonHandler::handle(const SendRawTxHex::Request& req, SendRawTxHex::Response& res) + { + std::string tx_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(req.tx_as_hex, tx_blob)) + { + MERROR("[SendRawTxHex]: Failed to parse tx from hexbuff: " << req.tx_as_hex); + res.status = Message::STATUS_FAILED; + res.error_details = "Invalid hex"; + return; + } + handleTxBlob(tx_blob, req.relay, res); + } + + void DaemonHandler::handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res) + { + if (!m_p2p.get_payload_object().is_synchronized()) + { + res.status = Message::STATUS_FAILED; + res.error_details = "Not ready to accept transactions; try again later"; + return; + } cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !req.relay) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !relay) || tvc.m_verifivation_failed) { if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed"); + MERROR("[SendRawTx]: tx verification failed"); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx"); + MERROR("[SendRawTx]: Failed to process tx"); } res.status = Message::STATUS_FAILED; res.error_details = ""; @@ -328,9 +351,9 @@ namespace rpc return; } - if(!tvc.m_should_be_relayed || !req.relay) + if(!tvc.m_should_be_relayed || !relay) { - LOG_PRINT_L0("[on_send_raw_tx]: tx accepted, but not relayed"); + MERROR("[SendRawTx]: tx accepted, but not relayed"); res.error_details = "Not relayed"; res.relayed = false; res.status = Message::STATUS_OK; @@ -413,7 +436,8 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); - res.info.difficulty = chain.get_difficulty_for_next_block(); + res.info.wide_difficulty = chain.get_difficulty_for_next_block(); + res.info.difficulty = (res.info.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); res.info.target = chain.get_difficulty_target(); @@ -434,7 +458,8 @@ namespace rpc res.info.mainnet = m_core.get_nettype() == MAINNET; res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; - res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); + res.info.wide_cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); + res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); @@ -803,7 +828,8 @@ namespace rpc header.reward += out.amount; } - header.difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); + header.wide_difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); + header.difficulty = (header.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); return true; } @@ -830,6 +856,7 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, SendRawTxHex, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StopMining, req_json, resp_message, handle); diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 87161784e..34723f676 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -68,6 +68,8 @@ class DaemonHandler : public RpcHandler void handle(const SendRawTx::Request& req, SendRawTx::Response& res); + void handle(const SendRawTxHex::Request& req, SendRawTxHex::Response& res); + void handle(const StartMining::Request& req, StartMining::Response& res); void handle(const GetInfo::Request& req, GetInfo::Response& res); @@ -136,6 +138,8 @@ class DaemonHandler : public RpcHandler bool getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& response); + void handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res); + cryptonote::core& m_core; t_p2p& m_p2p; }; diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index ecd3683e5..cf0f5ece1 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -42,6 +42,7 @@ const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; const char* const SendRawTx::name = "send_raw_tx"; +const char* const SendRawTxHex::name = "send_raw_tx_hex"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; const char* const MiningStatus::name = "mining_status"; @@ -292,6 +293,22 @@ void SendRawTx::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, relayed, relayed); } +rapidjson::Value SendRawTxHex::Request::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + + INSERT_INTO_JSON_OBJECT(val, doc, tx_as_hex, tx_as_hex); + INSERT_INTO_JSON_OBJECT(val, doc, relay, relay); + + return val; +} + +void SendRawTxHex::Request::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, tx_as_hex, tx_as_hex); + GET_FROM_JSON_OBJECT(val, relay, relay); +} + rapidjson::Value StartMining::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 8ce56b6af..c0d9aed0a 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -171,6 +171,14 @@ BEGIN_RPC_MESSAGE_CLASS(SendRawTx); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; +BEGIN_RPC_MESSAGE_CLASS(SendRawTxHex); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::string, tx_as_hex); + RPC_MESSAGE_MEMBER(bool, relay); + END_RPC_MESSAGE_REQUEST; + using Response = SendRawTx::Response; +END_RPC_MESSAGE_CLASS; + BEGIN_RPC_MESSAGE_CLASS(StartMining); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(std::string, miner_address); diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 26c5038f6..2a43811cf 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -30,6 +30,7 @@ #include "crypto/hash.h" #include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_basic/difficulty.h" #include "ringct/rctSigs.h" #include "rpc/rpc_handler.h" @@ -165,6 +166,7 @@ namespace rpc uint64_t height; uint64_t depth; crypto::hash hash; + cryptonote::difficulty_type wide_difficulty; uint64_t difficulty; uint64_t reward; }; @@ -173,6 +175,7 @@ namespace rpc { uint64_t height; uint64_t target_height; + cryptonote::difficulty_type wide_difficulty; uint64_t difficulty; uint64_t target; uint64_t tx_count; @@ -187,6 +190,7 @@ namespace rpc bool stagenet; std::string nettype; crypto::hash top_block_hash; + cryptonote::difficulty_type wide_cumulative_difficulty; uint64_t cumulative_difficulty; uint64_t block_size_limit; uint64_t block_weight_limit; diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h new file mode 100644 index 000000000..e32e24b78 --- /dev/null +++ b/src/serialization/difficulty_type.h @@ -0,0 +1,65 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_basic/difficulty.h" +#include "serialization.h" + +template<> struct is_basic_type<cryptonote::difficulty_type> { typedef boost::true_type type; }; + +template <template <bool> class Archive> +inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) +{ + uint64_t hi, lo; + ar.serialize_varint(hi); + if (!ar.stream().good()) + return false; + ar.serialize_varint(lo); + if (!ar.stream().good()) + return false; + diff = hi; + diff <<= 64; + diff += lo; + return true; +} + +template <template <bool> class Archive> +inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff) +{ + if (!ar.stream().good()) + return false; + const uint64_t hi = (diff >> 64).convert_to<uint64_t>(); + const uint64_t lo = (diff << 64 >> 64).convert_to<uint64_t>(); + ar.serialize_varint(hi); + ar.serialize_varint(lo); + if (!ar.stream().good()) + return false; + return true; +} + diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 3c89c68a0..91a8f6348 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -116,7 +116,9 @@ typedef cryptonote::simple_wallet sw; #define LONG_PAYMENT_ID_SUPPORT_CHECK() \ do { \ if (!m_long_payment_id_support) { \ - fail_msg_writer() << tr("Long payment IDs are obsolete. Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ + fail_msg_writer() << tr("Warning: Long payment IDs are obsolete."); \ + fail_msg_writer() << tr("Long payment IDs are not encrypted on the blockchain, and will harm your privacy."); \ + fail_msg_writer() << tr("Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ return true; \ } \ } while(0) @@ -149,7 +151,7 @@ namespace const command_line::arg_descriptor<bool> arg_create_address_file = {"create-address-file", sw::tr("Create an address file for new wallets"), false}; const command_line::arg_descriptor<std::string> arg_subaddress_lookahead = {"subaddress-lookahead", tools::wallet2::tr("Set subaddress lookahead sizes to <major>:<minor>"), ""}; const command_line::arg_descriptor<bool> arg_use_english_language_names = {"use-english-language-names", sw::tr("Display English language names"), false}; - const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support", sw::tr("Support obsolete long (unencrypted) payment ids"), false}; + const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support-bad-for-privacy", sw::tr("Support obsolete long (unencrypted) payment ids (using them harms your privacy)"), false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -2049,7 +2051,7 @@ bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux); // import key images - return m_wallet->import_key_images(exported_txs.key_images); + return m_wallet->import_key_images(exported_txs, 0, true); } bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -4686,12 +4688,12 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_button_request() +void simple_wallet::on_device_button_request(uint64_t code) { message_writer(console_color_white, false) << tr("Device requires attention"); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request() { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4699,14 +4701,14 @@ void simple_wallet::on_pin_request(epee::wipeable_string & pin) std::string msg = tr("Enter device PIN"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN")); - pin = pwd_container->password(); + return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device) { if (on_device){ message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); - return; + return boost::none; } #ifdef HAVE_READLINE @@ -4715,13 +4717,13 @@ void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string std::string msg = tr("Enter device passphrase"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); - passphrase = pwd_container->password(); + return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money) { // Key image sync after the first refresh - if (!m_wallet->get_account().get_device().has_tx_cold_sign()) { + if (!m_wallet->get_account().get_device().has_tx_cold_sign() || m_wallet->get_account().get_device().has_ki_live_refresh()) { return; } @@ -4981,7 +4983,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args std::vector<uint64_t> heights; for (const auto &e: td.m_uses) heights.push_back(e.first); const std::pair<std::string, std::string> line = show_outputs_line(heights, blockchain_height, td.m_spent_height); - extra_string += tr("Heights: ") + line.first + "\n" + line.second; + extra_string += std::string("\n ") + tr("Used at heights: ") + line.first + "\n " + line.second; } message_writer(td.m_spent ? console_color_magenta : console_color_green, false) << boost::format("%21s%8s%12s%8s%16u%68s%16u%s") % @@ -5355,7 +5357,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri r = add_extra_nonce_to_tx_extra(extra, extra_nonce); local_args.pop_back(); payment_id_seen = true; - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + message_writer() << tr("Warning: Unencrypted payment IDs will harm your privacy: ask the recipient to use subaddresses instead"); } if(!r) { @@ -5465,7 +5467,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri { LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + message_writer() << tr("Warning: Unencrypted payment IDs will harm your privacy: ask the recipient to use subaddresses instead"); } else { @@ -6772,7 +6774,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) { std::vector<std::string> local_args = args_; - if (m_wallet->key_on_device()) + if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; @@ -6793,7 +6795,9 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; - if (m_wallet->get_tx_key(txid, tx_key, additional_tx_keys)) + + bool found_tx_key = m_wallet->get_tx_key(txid, tx_key, additional_tx_keys); + if (found_tx_key) { ostringstream oss; oss << epee::string_tools::pod_to_hex(tx_key); @@ -6869,7 +6873,7 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) { - if (m_wallet->key_on_device()) + if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; @@ -8655,13 +8659,8 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) { uint64_t spent = 0, unspent = 0; uint64_t height = m_wallet->import_key_images(filename, spent, unspent); - if (height > 0) - { - success_msg_writer() << "Signed key images imported to height " << height << ", " - << print_money(spent) << " spent, " << print_money(unspent) << " unspent"; - } else { - fail_msg_writer() << "Failed to import key images"; - } + success_msg_writer() << "Signed key images imported to height " << height << ", " + << print_money(spent) << " spent, " << print_money(unspent) << " unspent"; } catch (const std::exception &e) { @@ -10012,6 +10011,16 @@ bool simple_wallet::mms(const std::vector<std::string> &args) { try { + m_wallet->get_multisig_wallet_state(); + } + catch(const std::exception &e) + { + fail_msg_writer() << tr("MMS not available in this wallet"); + return true; + } + + try + { mms::message_store& ms = m_wallet->get_message_store(); if (args.size() == 0) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index cf5458175..7bcb92190 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -300,9 +300,9 @@ namespace cryptonote virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); - virtual void on_button_request(); - virtual void on_pin_request(epee::wipeable_string & pin); - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + virtual void on_device_button_request(uint64_t code); + virtual boost::optional<epee::wipeable_string> on_device_pin_request(); + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); //---------------------------------------------------------- friend class refresh_progress_reporter_t; diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index efd61cb5a..def23aff0 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -63,6 +63,7 @@ target_link_libraries(wallet cryptonote_core mnemonics device_trezor + net ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index e649f1f3a..52510164a 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -109,6 +109,23 @@ bool PendingTransactionImpl::commit(const std::string &filename, bool overwrite) } m_wallet.pauseRefresh(); + + const bool tx_cold_signed = m_wallet.m_wallet->get_account().get_device().has_tx_cold_sign(); + if (tx_cold_signed){ + std::unordered_set<size_t> selected_transfers; + for(const tools::wallet2::pending_tx & ptx : m_pending_tx){ + for(size_t s : ptx.selected_transfers){ + selected_transfers.insert(s); + } + } + + m_wallet.m_wallet->cold_tx_aux_import(m_pending_tx, m_tx_device_aux); + bool r = m_wallet.m_wallet->import_key_images(m_key_images, 0, selected_transfers); + if (!r){ + throw runtime_error("Cold sign transaction submit failed - key image sync fail"); + } + } + while (!m_pending_tx.empty()) { auto & ptx = m_pending_tx.back(); m_wallet.m_wallet->commit_tx(ptx); diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index 4ec7c656a..92801d77d 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -67,6 +67,8 @@ private: std::string m_errorString; std::vector<tools::wallet2::pending_tx> m_pending_tx; std::unordered_set<crypto::public_key> m_signers; + std::vector<std::string> m_tx_device_aux; + std::vector<crypto::key_image> m_key_images; }; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index da6ddc8a3..82986ba2d 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -242,6 +242,42 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback } } + virtual void on_device_button_request(uint64_t code) + { + if (m_listener) { + m_listener->onDeviceButtonRequest(code); + } + } + + virtual boost::optional<epee::wipeable_string> on_device_pin_request() + { + if (m_listener) { + auto pin = m_listener->onDevicePinRequest(); + if (pin){ + return boost::make_optional(epee::wipeable_string((*pin).data(), (*pin).size())); + } + } + return boost::none; + } + + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) + { + if (m_listener) { + auto passphrase = m_listener->onDevicePassphraseRequest(on_device); + if (!on_device && passphrase) { + return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size())); + } + } + return boost::none; + } + + virtual void on_device_progress(const hw::device_progress & event) + { + if (m_listener) { + m_listener->onDeviceProgress(DeviceProgress(event.progress(), event.indeterminate())); + } + } + WalletListener * m_listener; WalletImpl * m_wallet; }; @@ -785,6 +821,28 @@ bool WalletImpl::setPassword(const std::string &password) return status() == Status_Ok; } +bool WalletImpl::setDevicePin(const std::string &pin) +{ + clearStatus(); + try { + m_wallet->get_account().get_device().set_pin(epee::wipeable_string(pin.data(), pin.size())); + } catch (const std::exception &e) { + setStatusError(e.what()); + } + return status() == Status_Ok; +} + +bool WalletImpl::setDevicePassphrase(const std::string &passphrase) +{ + clearStatus(); + try { + m_wallet->get_account().get_device().set_passphrase(epee::wipeable_string(passphrase.data(), passphrase.size())); + } catch (const std::exception &e) { + setStatusError(e.what()); + } + return status() == Status_Ok; +} + std::string WalletImpl::address(uint32_t accountIndex, uint32_t addressIndex) const { return m_wallet->get_subaddress_as_str({accountIndex, addressIndex}); @@ -1428,8 +1486,12 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const extra, subaddr_account, subaddr_indices); } + pendingTxPostProcess(transaction); + if (multisig().isMultisig) { - transaction->m_signers = m_wallet->make_multisig_tx_set(transaction->m_pending_tx).m_signers; + auto tx_set = m_wallet->make_multisig_tx_set(transaction->m_pending_tx); + transaction->m_pending_tx = tx_set.m_ptx; + transaction->m_signers = tx_set.m_signers; } } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1511,6 +1573,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); + pendingTxPostProcess(transaction); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -2093,10 +2156,24 @@ bool WalletImpl::isNewWallet() const return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly(); } +void WalletImpl::pendingTxPostProcess(PendingTransactionImpl * pending) +{ + // If the device being used is HW device with cold signing protocol, cold sign then. + if (!m_wallet->get_account().get_device().has_tx_cold_sign()){ + return; + } + + tools::wallet2::signed_tx_set exported_txs; + std::vector<cryptonote::address_parse_info> dsts_info; + + m_wallet->cold_sign_tx(pending->m_pending_tx, exported_txs, dsts_info, pending->m_tx_device_aux); + pending->m_key_images = exported_txs.key_images; + pending->m_pending_tx = exported_txs.ptx; +} + bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) { - // claim RPC so there's no in-memory encryption for now - if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit, ssl)) + if (!m_wallet->init(daemon_address, m_daemon_login, tcp::endpoint{}, upper_transaction_size_limit)) return false; // in case new wallet, this will force fast-refresh (pulling hashes instead of blocks) @@ -2325,6 +2402,11 @@ bool WalletImpl::isKeysFileLocked() { return m_wallet->is_keys_file_locked(); } + +uint64_t WalletImpl::coldKeyImageSync(uint64_t &spent, uint64_t &unspent) +{ + return m_wallet->cold_key_image_sync(spent, unspent); +} } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index bd33b773c..9e07b6e19 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -89,6 +89,8 @@ public: std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; bool setPassword(const std::string &password) override; + bool setDevicePin(const std::string &password) override; + bool setDevicePassphrase(const std::string &password) override; std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; std::string integratedAddress(const std::string &payment_id) const override; std::string secretViewKey() const override; @@ -198,6 +200,7 @@ public: virtual bool lockKeysFile() override; virtual bool unlockKeysFile() override; virtual bool isKeysFileLocked() override; + virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) override; private: void clearStatus() const; @@ -209,6 +212,7 @@ private: bool daemonSynced() const; void stopRefresh(); bool isNewWallet() const; + void pendingTxPostProcess(PendingTransactionImpl * pending); bool doInit(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false); private: diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index c549c260b..ee1d6ae79 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -324,6 +324,19 @@ struct MultisigState { uint32_t total; }; + +struct DeviceProgress { + DeviceProgress(): m_progress(0), m_indeterminate(false) {} + DeviceProgress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate) {} + + virtual double progress() const { return m_progress; } + virtual bool indeterminate() const { return m_indeterminate; } + +protected: + double m_progress; + bool m_indeterminate; +}; + struct WalletListener { virtual ~WalletListener() = 0; @@ -364,6 +377,31 @@ struct WalletListener * @brief refreshed - called when wallet refreshed by background thread or explicitly refreshed by calling "refresh" synchronously */ virtual void refreshed() = 0; + + /** + * @brief called by device if the action is required + */ + virtual void onDeviceButtonRequest(uint64_t code) {} + + /** + * @brief called by device when PIN is needed + */ + virtual optional<std::string> onDevicePinRequest() { + throw std::runtime_error("Not supported"); + } + + /** + * @brief called by device when passphrase entry is needed + */ + virtual optional<std::string> onDevicePassphraseRequest(bool on_device) { + if (!on_device) throw std::runtime_error("Not supported"); + return optional<std::string>(); + } + + /** + * @brief Signalizes device operation progress + */ + virtual void onDeviceProgress(const DeviceProgress & event) {}; }; @@ -375,7 +413,8 @@ struct Wallet { enum Device { Device_Software = 0, - Device_Ledger = 1 + Device_Ledger = 1, + Device_Trezor = 2 }; enum Status { @@ -401,6 +440,8 @@ struct Wallet //! returns both error and error string atomically. suggested to use in instead of status() and errorString() virtual void statusWithErrorString(int& status, std::string& errorString) const = 0; virtual bool setPassword(const std::string &password) = 0; + virtual bool setDevicePin(const std::string &password) { return false; }; + virtual bool setDevicePassphrase(const std::string &password) { return false; }; virtual std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const = 0; std::string mainAddress() const { return address(0, 0); } virtual std::string path() const = 0; @@ -947,6 +988,9 @@ struct Wallet * \return Device they are on */ virtual Device getDeviceType() const = 0; + + //! cold-device protocol key image sync + virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) = 0; }; /** diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 0cedd4b50..439873932 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -38,6 +38,7 @@ #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> #include "include_base_utils.h" using namespace epee; @@ -75,6 +76,7 @@ using namespace epee; #include "ringdb.h" #include "device/device_cold.hpp" #include "device_trezor/device_trezor.hpp" +#include "net/socks_connect.h" extern "C" { @@ -231,6 +233,7 @@ namespace struct options { const command_line::arg_descriptor<std::string> daemon_address = {"daemon-address", tools::wallet2::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> daemon_host = {"daemon-host", tools::wallet2::tr("Use daemon instance at host <arg> instead of localhost"), ""}; + const command_line::arg_descriptor<std::string> proxy = {"proxy", tools::wallet2::tr("[<ip>:]<port> socks proxy to use for daemon connections"), {}, true}; const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; @@ -303,6 +306,8 @@ std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_siz std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { + namespace ip = boost::asio::ip; + const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; @@ -352,6 +357,44 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); + boost::asio::ip::tcp::endpoint proxy{}; + if (command_line::has_arg(vm, opts.proxy)) + { + namespace ip = boost::asio::ip; + const boost::string_ref real_daemon = boost::string_ref{daemon_address}.substr(0, daemon_address.rfind(':')); + + // onion and i2p addresses contain information about the server cert + // which both authenticates and encrypts + const bool unencrypted_proxy = + !real_daemon.ends_with(".onion") && !real_daemon.ends_with(".i2p") && + daemon_ssl_allowed_certificates.empty() && daemon_ssl_allowed_fingerprints.empty(); + THROW_WALLET_EXCEPTION_IF( + unencrypted_proxy, + tools::error::wallet_internal_error, + std::string{"Use of --"} + opts.proxy.name + " requires --" + opts.daemon_ssl_allowed_certificates.name + " or --" + opts.daemon_ssl_allowed_fingerprints.name + " or use of a .onion/.i2p domain" + ); + + const auto proxy_address = command_line::get_arg(vm, opts.proxy); + + boost::string_ref proxy_port{proxy_address}; + boost::string_ref proxy_host = proxy_port.substr(0, proxy_port.rfind(":")); + if (proxy_port.size() == proxy_host.size()) + proxy_host = "127.0.0.1"; + else + proxy_port = proxy_port.substr(proxy_host.size() + 1); + + uint16_t port_value = 0; + THROW_WALLET_EXCEPTION_IF( + !epee::string_tools::get_xtype_from_string(port_value, std::string{proxy_port}), + tools::error::wallet_internal_error, + std::string{"Invalid port specified for --"} + opts.proxy.name + ); + + boost::system::error_code error{}; + proxy = ip::tcp::endpoint{ip::address::from_string(std::string{proxy_host}, error), port_value}; + THROW_WALLET_EXCEPTION_IF(bool(error), tools::error::wallet_internal_error, std::string{"Invalid IP address specified for --"} + opts.proxy.name); + } + boost::optional<bool> trusted_daemon; if (!command_line::is_arg_defaulted(vm, opts.trusted_daemon) || !command_line::is_arg_defaulted(vm, opts.untrusted_daemon)) trusted_daemon = command_line::get_arg(vm, opts.trusted_daemon) && !command_line::get_arg(vm, opts.untrusted_daemon); @@ -388,8 +431,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl std::transform(daemon_ssl_allowed_fingerprints.begin(), daemon_ssl_allowed_fingerprints.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds, unattended)); - wallet->init(std::move(daemon_address), std::move(login), 0, *trusted_daemon, ssl_support, std::make_pair(daemon_ssl_private_key, daemon_ssl_certificate), ssl_allowed_certificates, ssl_allowed_fingerprints, daemon_ssl_allow_any_cert); - + wallet->init(std::move(daemon_address), std::move(login), std::move(proxy), 0, *trusted_daemon, ssl_support, std::make_pair(daemon_ssl_private_key, daemon_ssl_certificate), ssl_allowed_certificates, ssl_allowed_fingerprints, daemon_ssl_allow_any_cert); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); wallet->get_message_store().set_options(vm); @@ -470,7 +512,7 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, filename, std::string, String, true, std::string()); GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, scan_from_height, uint64_t, Uint64, false, 0); - const bool recover = field_scan_from_height_found; + const bool recover = true; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, password, std::string, String, false, std::string()); @@ -580,6 +622,8 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f wallet.reset(make_basic(vm, unattended, opts, password_prompter).release()); wallet->set_refresh_from_block_height(field_scan_from_height); wallet->explicit_refresh_from_block_height(field_scan_from_height_found); + if (!old_language.empty()) + wallet->set_seed_language(old_language); try { @@ -928,22 +972,30 @@ wallet_keys_unlocker::~wallet_keys_unlocker() } } -void wallet_device_callback::on_button_request() +void wallet_device_callback::on_button_request(uint64_t code) +{ + if (wallet) + wallet->on_device_button_request(code); +} + +boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request() { if (wallet) - wallet->on_button_request(); + return wallet->on_device_pin_request(); + return boost::none; } -void wallet_device_callback::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device) { if (wallet) - wallet->on_pin_request(pin); + return wallet->on_device_passphrase_request(on_device); + return boost::none; } -void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +void wallet_device_callback::on_progress(const hw::device_progress& event) { if (wallet) - wallet->on_passphrase_request(on_device, passphrase); + wallet->on_device_progress(event); } wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): @@ -1038,6 +1090,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par const options opts{}; command_line::add_arg(desc_params, opts.daemon_address); command_line::add_arg(desc_params, opts.daemon_host); + command_line::add_arg(desc_params, opts.proxy); command_line::add_arg(desc_params, opts.trusted_daemon); command_line::add_arg(desc_params, opts.untrusted_daemon); command_line::add_arg(desc_params, opts.password); @@ -1101,7 +1154,7 @@ std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::varia } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_support_t ssl_support, const std::pair<std::string, std::string> &private_key_and_certificate_path, const std::list<std::string> &allowed_certificates, const std::vector<std::vector<uint8_t>> &allowed_fingerprints, bool allow_any_cert) +bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, boost::asio::ip::tcp::endpoint proxy, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_support_t ssl_support, const std::pair<std::string, std::string> &private_key_and_certificate_path, const std::list<std::string> &allowed_certificates, const std::vector<std::vector<uint8_t>> &allowed_fingerprints, bool allow_any_cert) { m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) @@ -1111,6 +1164,10 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); m_trusted_daemon = trusted_daemon; + if (proxy != boost::asio::ip::tcp::endpoint{}) + m_http_client.set_connector(net::socks::connector{std::move(proxy)}); + + // When switching from light wallet to full wallet, we need to reset the height we got from lw node. return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl_support, private_key_and_certificate_path, allowed_certificates, allowed_fingerprints, allow_any_cert); } //---------------------------------------------------------------------------------------------------- @@ -2398,7 +2455,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry " (height " + std::to_string(start_height) + "), local block id at this height: " + string_tools::pod_to_hex(m_blockchain[current_index])); - detach_blockchain(current_index); + detach_blockchain(current_index, output_tracker_cache); process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset, output_tracker_cache); } else @@ -2842,7 +2899,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. // This call is not really needed for other purposes and can be removed if mymonero changes their backend. - cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::response res; + tools::COMMAND_RPC_GET_ADDRESS_INFO::response res; // Get basic info if(light_wallet_get_address_info(res)) { @@ -2883,6 +2940,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo std::vector<parsed_block> parsed_blocks; bool refreshed = false; std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> output_tracker_cache; + hw::device &hwdev = m_account.get_device(); // pull the first set of blocks get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); @@ -2989,7 +3047,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo // if we've got at least 10 blocks to refresh, assume we're starting // a long refresh, and setup a tracking output cache if we need to - if (m_track_uses && !output_tracker_cache && next_blocks.size() >= 10) + if (m_track_uses && (!output_tracker_cache || output_tracker_cache->empty()) && next_blocks.size() >= 10) output_tracker_cache = create_output_tracker_cache(); // switch to the new blocks from the daemon @@ -3039,6 +3097,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo LOG_PRINT_L1("Failed to check pending transactions"); } + hwdev.computing_key_images(false); m_first_refresh_done = true; LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); @@ -3129,7 +3188,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> return true; } //---------------------------------------------------------------------------------------------------- -void wallet2::detach_blockchain(uint64_t height) +void wallet2::detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache) { LOG_PRINT_L0("Detaching blockchain on height " << height); @@ -3151,6 +3210,15 @@ void wallet2::detach_blockchain(uint64_t height) } } + for (transfer_details &td: m_transfers) + { + while (!td.m_uses.empty() && td.m_uses.back().first >= height) + td.m_uses.pop_back(); + } + + if (output_tracker_cache) + output_tracker_cache->clear(); + auto it = std::find_if(m_transfers.begin(), m_transfers.end(), [&](const transfer_details& td){return td.m_block_height >= height;}); size_t i_start = it - m_transfers.begin(); @@ -3480,7 +3548,8 @@ void wallet2::change_password(const std::string &filename, const epee::wipeable_ decrypt_keys(original_password); setup_keys(new_password); rewrite(filename, new_password); - store(); + if (!filename.empty()) + store(); } //---------------------------------------------------------------------------------------------------- /*! @@ -5110,7 +5179,14 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass MERROR("Failed to save rings, will try again next time"); } - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + try + { + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + } + catch (const std::exception &e) + { + MERROR("Failed to initialize MMS, it will be unusable"); + } } //---------------------------------------------------------------------------------------------------- void wallet2::trim_hashchain() @@ -5162,7 +5238,8 @@ std::string wallet2::path() const //---------------------------------------------------------------------------------------------------- void wallet2::store() { - store_to("", epee::wipeable_string()); + if (!m_wallet_file.empty()) + store_to("", epee::wipeable_string()); } //---------------------------------------------------------------------------------------------------- void wallet2::store_to(const std::string &path, const epee::wipeable_string &password) @@ -5995,12 +6072,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; - if (sd.use_bulletproofs) - { - rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; - rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; - } + rct::RCTConfig rct_config = sd.rct_config; crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; @@ -6477,12 +6549,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; - if (sd.use_bulletproofs) - { - rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; - rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; - } + rct::RCTConfig rct_config = sd.rct_config; bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, rct_config, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); @@ -7050,6 +7117,17 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; + // check the keys are valid + if (!rct::isInMainSubgroup(rct::pk2rct(output_public_key))) + { + MWARNING("Key " << output_public_key << " at index " << global_index << " is not in the main subgroup"); + return false; + } + if (!rct::isInMainSubgroup(mask)) + { + MWARNING("Commitment " << mask << " at index " << global_index << " is not in the main subgroup"); + return false; + } // if (is_output_blackballed(output_public_key)) // don't add blackballed outputs // return false; outs.back().push_back(item); @@ -7932,7 +8010,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; - ptx.construction_data.use_bulletproofs = false; + ptx.construction_data.rct_config = { rct::RangeProofBorromean, 0 }; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -8118,6 +8196,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry else { change_dts.addr = get_subaddress({subaddr_account, 0}); + change_dts.is_subaddress = subaddr_account != 0; splitted_dsts.push_back(change_dts); } @@ -8214,7 +8293,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = true; - ptx.construction_data.use_bulletproofs = !tx.rct_signatures.p.bulletproofs.empty(); + ptx.construction_data.rct_config = { tx.rct_signatures.p.bulletproofs.empty() ? rct::RangeProofBorromean : rct::RangeProofPaddedBulletproof, use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1}; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -8342,8 +8421,8 @@ bool wallet2::light_wallet_login(bool &new_address) { MDEBUG("Light wallet login request"); m_light_wallet_connected = false; - cryptonote::COMMAND_RPC_LOGIN::request request; - cryptonote::COMMAND_RPC_LOGIN::response response; + tools::COMMAND_RPC_LOGIN::request request; + tools::COMMAND_RPC_LOGIN::response response; request.address = get_account().get_public_address_str(m_nettype); request.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); // Always create account if it doesn't exist. @@ -8367,10 +8446,10 @@ bool wallet2::light_wallet_login(bool &new_address) return m_light_wallet_connected; } -bool wallet2::light_wallet_import_wallet_request(cryptonote::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response) +bool wallet2::light_wallet_import_wallet_request(tools::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response) { MDEBUG("Light wallet import wallet request"); - cryptonote::COMMAND_RPC_IMPORT_WALLET_REQUEST::request oreq; + tools::COMMAND_RPC_IMPORT_WALLET_REQUEST::request oreq; oreq.address = get_account().get_public_address_str(m_nettype); oreq.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); m_daemon_rpc_mutex.lock(); @@ -8386,8 +8465,8 @@ void wallet2::light_wallet_get_unspent_outs() { MDEBUG("Getting unspent outs"); - cryptonote::COMMAND_RPC_GET_UNSPENT_OUTS::request oreq; - cryptonote::COMMAND_RPC_GET_UNSPENT_OUTS::response ores; + tools::COMMAND_RPC_GET_UNSPENT_OUTS::request oreq; + tools::COMMAND_RPC_GET_UNSPENT_OUTS::response ores; oreq.amount = "0"; oreq.address = get_account().get_public_address_str(m_nettype); @@ -8535,11 +8614,11 @@ void wallet2::light_wallet_get_unspent_outs() } } -bool wallet2::light_wallet_get_address_info(cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::response &response) +bool wallet2::light_wallet_get_address_info(tools::COMMAND_RPC_GET_ADDRESS_INFO::response &response) { MTRACE(__FUNCTION__); - cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::request request; + tools::COMMAND_RPC_GET_ADDRESS_INFO::request request; request.address = get_account().get_public_address_str(m_nettype); request.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); @@ -8555,8 +8634,8 @@ void wallet2::light_wallet_get_address_txs() { MDEBUG("Refreshing light wallet"); - cryptonote::COMMAND_RPC_GET_ADDRESS_TXS::request ireq; - cryptonote::COMMAND_RPC_GET_ADDRESS_TXS::response ires; + tools::COMMAND_RPC_GET_ADDRESS_TXS::request ireq; + tools::COMMAND_RPC_GET_ADDRESS_TXS::response ires; ireq.address = get_account().get_public_address_str(m_nettype); ireq.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); @@ -8829,6 +8908,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp boost::unique_lock<hw::device> hwdev_lock (hwdev); hw::reset_mode rst(hwdev); + auto original_dsts = dsts; + if(m_light_wallet) { // Populate m_transfers light_wallet_get_unspent_outs(); @@ -9349,10 +9430,77 @@ skip_tx: ptx_vector.push_back(tx.ptx); } + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, original_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + // if we made it this far, we're OK to actually send the transactions return ptx_vector; } +bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const +{ + MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations"); + + hw::device &hwdev = m_account.get_device(); + + THROW_WALLET_EXCEPTION_IF(ptx_vector.empty(), error::wallet_internal_error, "No transactions"); + + // check every party in there does receive at least the required amount + std::unordered_map<account_public_address, std::pair<uint64_t, bool>> required; + for (const auto &d: dsts) + { + required[d.addr].first += d.amount; + required[d.addr].second = d.is_subaddress; + } + + // add change + uint64_t change = 0; + for (const auto &ptx: ptx_vector) + { + for (size_t idx: ptx.selected_transfers) + change += m_transfers[idx].amount(); + change -= ptx.fee; + } + for (const auto &r: required) + change -= r.second.first; + MDEBUG("Adding " << cryptonote::print_money(change) << " expected change"); + + for (const pending_tx &ptx: ptx_vector) + THROW_WALLET_EXCEPTION_IF(ptx.change_dts.addr != ptx_vector[0].change_dts.addr, error::wallet_internal_error, + "Change goes to several different addresses"); + const auto it = m_subaddresses.find(ptx_vector[0].change_dts.addr.m_spend_public_key); + THROW_WALLET_EXCEPTION_IF(it == m_subaddresses.end(), error::wallet_internal_error, "Change address is not ours"); + + required[ptx_vector[0].change_dts.addr].first += change; + required[ptx_vector[0].change_dts.addr].second = ptx_vector[0].change_dts.is_subaddress; + + for (const auto &r: required) + { + const account_public_address &address = r.first; + const crypto::public_key &view_pkey = address.m_view_public_key; + + uint64_t total_received = 0; + for (const auto &ptx: ptx_vector) + { + uint64_t received = 0; + try + { + std::string proof = get_tx_proof(ptx.tx, ptx.tx_key, ptx.additional_tx_keys, address, r.second.second, "automatic-sanity-check"); + check_tx_proof(ptx.tx, address, r.second.second, "automatic-sanity-check", proof, received); + } + catch (const std::exception &e) { received = 0; } + total_received += received; + } + + std::stringstream ss; + ss << "Total received by " << cryptonote::get_account_address_as_str(m_nettype, r.second.second, address) << ": " + << cryptonote::print_money(total_received) << ", expected " << cryptonote::print_money(r.second.first); + MDEBUG(ss.str()); + THROW_WALLET_EXCEPTION_IF(total_received < r.second.first, error::wallet_internal_error, ss.str()); + } + + return true; +} + std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; @@ -9635,6 +9783,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton ptx_vector.push_back(tx.ptx); } + uint64_t a = 0; + for (size_t idx: unused_transfers_indices) a += m_transfers[idx].amount(); + for (size_t idx: unused_dust_indices) a += m_transfers[idx].amount(); + std::vector<cryptonote::tx_destination_entry> synthetic_dsts(1, cryptonote::tx_destination_entry("", a, address, is_subaddress)); + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, synthetic_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + // if we made it this far, we're OK to actually send the transactions return ptx_vector; } @@ -9670,6 +9824,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_ hw::wallet_shim wallet_shim; setup_shim(&wallet_shim, this); aux_data.tx_recipients = dsts_info; + aux_data.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); tx_device_aux = aux_data.tx_device_aux; @@ -9714,7 +9869,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand + bool close_enough = (int64_t)height >= (int64_t)earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -9886,7 +10041,7 @@ void wallet2::discard_unmixable_outputs() } } -bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const +bool wallet2::get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const { additional_tx_keys.clear(); const std::unordered_map<crypto::hash, crypto::secret_key>::const_iterator i = m_tx_keys.find(txid); @@ -9899,6 +10054,82 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s return true; } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) +{ + bool r = get_tx_key_cached(txid, tx_key, additional_tx_keys); + if (r) + { + return true; + } + + auto & hwdev = get_account().get_device(); + + // So far only Cold protocol devices are supported. + if (hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + { + return false; + } + + const auto tx_data_it = m_tx_device.find(txid); + if (tx_data_it == m_tx_device.end()) + { + MDEBUG("Aux data not found for txid: " << txid); + return false; + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + if (!dev_cold->is_get_tx_key_supported()) + { + MDEBUG("get_tx_key not supported by the device"); + return false; + } + + hw::device_cold::tx_key_data_t tx_key_data; + dev_cold->load_tx_key_data(tx_key_data, tx_data_it->second); + + // Load missing tx prefix hash + if (tx_key_data.tx_prefix_hash.empty()) + { + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash{}; + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash{}; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, + "Failed to get the right transaction from daemon"); + + tx_key_data.tx_prefix_hash = std::string(tx_prefix_hash.data, 32); + } + + std::vector<crypto::secret_key> tx_keys; + dev_cold->get_tx_key(tx_keys, tx_key_data, m_account.get_keys().m_view_secret_key); + if (tx_keys.empty()) + { + return false; + } + + tx_key = tx_keys[0]; + tx_keys.erase(tx_keys.begin()); + additional_tx_keys = tx_keys; + + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys) { // fetch tx from daemon and check if secret keys agree with corresponding public keys @@ -10190,41 +10421,8 @@ void wallet2::check_tx_key(const crypto::hash &txid, const crypto::secret_key &t check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); } -void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) +void wallet2::check_tx_key_helper(const cryptonote::transaction &tx, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received) const { - COMMAND_RPC_GET_TRANSACTIONS::request req; - COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); - req.decode_as_json = false; - req.prune = true; - m_daemon_rpc_mutex.lock(); - bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), - error::wallet_internal_error, "Failed to get transaction from daemon"); - - cryptonote::transaction tx; - crypto::hash tx_hash; - if (res.txs.size() == 1) - { - ok = get_pruned_tx(res.txs.front(), tx, tx_hash); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - } - else - { - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - tx_hash = cryptonote::get_transaction_hash(tx); - } - - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, - "Failed to get the right transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, - "The size of additional derivations is wrong"); - received = 0; hw::device &hwdev = m_account.get_device(); for (size_t n = 0; n < tx.vout.size(); ++n) @@ -10272,6 +10470,44 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de received += amount; } } +} + +void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) +{ + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash; + if (res.txs.size() == 1) + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } + else + { + cryptonote::blobdata tx_data; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } + + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, + "Failed to get the right transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, + "The size of additional derivations is wrong"); + + check_tx_key_helper(tx, derivation, additional_derivations, address, received); in_pool = res.txs.front().in_pool; confirmations = (uint64_t)-1; @@ -10286,9 +10522,55 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) { + // fetch tx pubkey from the daemon + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash; + if (res.txs.size() == 1) + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } + else + { + cryptonote::blobdata tx_data; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } + + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); + + // determine if the address is found in the subaddress hash table (i.e. whether the proof is outbound or inbound) + crypto::secret_key tx_key = crypto::null_skey; + std::vector<crypto::secret_key> additional_tx_keys; + const bool is_out = m_subaddresses.count(address.m_spend_public_key) == 0; + if (is_out) + { + THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file."); + } + + return get_tx_proof(tx, tx_key, additional_tx_keys, address, is_subaddress, message); +} + +std::string wallet2::get_tx_proof(const cryptonote::transaction &tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) const +{ // determine if the address is found in the subaddress hash table (i.e. whether the proof is outbound or inbound) const bool is_out = m_subaddresses.count(address.m_spend_public_key) == 0; + const crypto::hash txid = cryptonote::get_transaction_hash(tx); std::string prefix_data((const char*)&txid, sizeof(crypto::hash)); prefix_data += message; crypto::hash prefix_hash; @@ -10299,10 +10581,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac std::string sig_str; if (is_out) { - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file."); - const size_t num_sigs = 1 + additional_tx_keys.size(); shared_secret.resize(num_sigs); sig.resize(num_sigs); @@ -10337,37 +10615,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac } else { - // fetch tx pubkey from the daemon - COMMAND_RPC_GET_TRANSACTIONS::request req; - COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); - req.decode_as_json = false; - req.prune = true; - m_daemon_rpc_mutex.lock(); - bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), - error::wallet_internal_error, "Failed to get transaction from daemon"); - - cryptonote::transaction tx; - crypto::hash tx_hash; - if (res.txs.size() == 1) - { - ok = get_pruned_tx(res.txs.front(), tx, tx_hash); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - } - else - { - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - tx_hash = cryptonote::get_transaction_hash(tx); - } - - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); - crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); THROW_WALLET_EXCEPTION_IF(tx_pub_key == null_pkey, error::wallet_internal_error, "Tx pubkey was not found"); @@ -10409,9 +10656,7 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac for (size_t i = 1; i < num_sigs; ++i) THROW_WALLET_EXCEPTION_IF(!crypto::generate_key_derivation(shared_secret[i], rct::rct2sk(rct::I), additional_derivations[i - 1]), error::wallet_internal_error, "Failed to generate key derivation"); uint64_t received; - bool in_pool; - uint64_t confirmations; - check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); + check_tx_key_helper(tx, derivation, additional_derivations, address, received); THROW_WALLET_EXCEPTION_IF(!received, error::wallet_internal_error, tr("No funds received in this tx.")); // concatenate all signature strings @@ -10424,37 +10669,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received, bool &in_pool, uint64_t &confirmations) { - const bool is_out = sig_str.substr(0, 3) == "Out"; - const std::string header = is_out ? "OutProofV1" : "InProofV1"; - const size_t header_len = header.size(); - THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, - "Signature header check error"); - - // decode base58 - std::vector<crypto::public_key> shared_secret(1); - std::vector<crypto::signature> sig(1); - const size_t pk_len = tools::base58::encode(std::string((const char *)&shared_secret[0], sizeof(crypto::public_key))).size(); - const size_t sig_len = tools::base58::encode(std::string((const char *)&sig[0], sizeof(crypto::signature))).size(); - const size_t num_sigs = (sig_str.size() - header_len) / (pk_len + sig_len); - THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * (pk_len + sig_len), error::wallet_internal_error, - "Wrong signature size"); - shared_secret.resize(num_sigs); - sig.resize(num_sigs); - for (size_t i = 0; i < num_sigs; ++i) - { - std::string pk_decoded; - std::string sig_decoded; - const size_t offset = header_len + i * (pk_len + sig_len); - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset, pk_len), pk_decoded), error::wallet_internal_error, - "Signature decoding error"); - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset + pk_len, sig_len), sig_decoded), error::wallet_internal_error, - "Signature decoding error"); - THROW_WALLET_EXCEPTION_IF(sizeof(crypto::public_key) != pk_decoded.size() || sizeof(crypto::signature) != sig_decoded.size(), error::wallet_internal_error, - "Signature decoding error"); - memcpy(&shared_secret[i], pk_decoded.data(), sizeof(crypto::public_key)); - memcpy(&sig[i], sig_decoded.data(), sizeof(crypto::signature)); - } - // fetch tx pubkey from the daemon COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; @@ -10486,12 +10700,62 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); + if (!check_tx_proof(tx, address, is_subaddress, message, sig_str, received)) + return false; + + in_pool = res.txs.front().in_pool; + confirmations = (uint64_t)-1; + if (!in_pool) + { + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (err.empty()) + confirmations = bc_height - (res.txs.front().block_height + 1); + } + + return true; +} + +bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const +{ + const bool is_out = sig_str.substr(0, 3) == "Out"; + const std::string header = is_out ? "OutProofV1" : "InProofV1"; + const size_t header_len = header.size(); + THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, + "Signature header check error"); + + // decode base58 + std::vector<crypto::public_key> shared_secret(1); + std::vector<crypto::signature> sig(1); + const size_t pk_len = tools::base58::encode(std::string((const char *)&shared_secret[0], sizeof(crypto::public_key))).size(); + const size_t sig_len = tools::base58::encode(std::string((const char *)&sig[0], sizeof(crypto::signature))).size(); + const size_t num_sigs = (sig_str.size() - header_len) / (pk_len + sig_len); + THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * (pk_len + sig_len), error::wallet_internal_error, + "Wrong signature size"); + shared_secret.resize(num_sigs); + sig.resize(num_sigs); + for (size_t i = 0; i < num_sigs; ++i) + { + std::string pk_decoded; + std::string sig_decoded; + const size_t offset = header_len + i * (pk_len + sig_len); + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset, pk_len), pk_decoded), error::wallet_internal_error, + "Signature decoding error"); + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset + pk_len, sig_len), sig_decoded), error::wallet_internal_error, + "Signature decoding error"); + THROW_WALLET_EXCEPTION_IF(sizeof(crypto::public_key) != pk_decoded.size() || sizeof(crypto::signature) != sig_decoded.size(), error::wallet_internal_error, + "Signature decoding error"); + memcpy(&shared_secret[i], pk_decoded.data(), sizeof(crypto::public_key)); + memcpy(&sig[i], sig_decoded.data(), sizeof(crypto::signature)); + } + crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); THROW_WALLET_EXCEPTION_IF(tx_pub_key == null_pkey, error::wallet_internal_error, "Tx pubkey was not found"); std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); THROW_WALLET_EXCEPTION_IF(additional_tx_pub_keys.size() + 1 != num_sigs, error::wallet_internal_error, "Signature size mismatch with additional tx pubkeys"); + const crypto::hash txid = cryptonote::get_transaction_hash(tx); std::string prefix_data((const char*)&txid, sizeof(crypto::hash)); prefix_data += message; crypto::hash prefix_hash; @@ -10538,7 +10802,7 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account if (good_signature[i]) THROW_WALLET_EXCEPTION_IF(!crypto::generate_key_derivation(shared_secret[i], rct::rct2sk(rct::I), additional_derivations[i - 1]), error::wallet_internal_error, "Failed to generate key derivation"); - check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); + check_tx_key_helper(tx, derivation, additional_derivations, address, received); return true; } return false; @@ -11040,15 +11304,6 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle size_t pk_index = 0; hw::device &hwdev = m_account.get_device(); - const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); - std::vector<crypto::key_derivation> additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) - { - additional_derivations.push_back({}); - bool r = hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back()); - THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key derivation"); - } - while (find_tx_extra_field_by_type(tx_extra_fields, pub_key_field, pk_index++)) { const crypto::public_key tx_pub_key = pub_key_field.pub_key; crypto::key_derivation derivation; @@ -11058,16 +11313,15 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle for (size_t i = 0; i < td.m_tx.vout.size(); ++i) { tx_scan_info_t tx_scan_info; - check_acc_out_precomp(td.m_tx.vout[i], derivation, additional_derivations, i, tx_scan_info); + check_acc_out_precomp(td.m_tx.vout[i], derivation, {}, i, tx_scan_info); if (!tx_scan_info.error && tx_scan_info.received) return tx_pub_key; } } - // we found no key yielding an output - THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, - "Public key yielding at least one output wasn't found in the transaction extra"); - return crypto::null_pkey; + // we found no key yielding an output, but it might be in the additional + // tx pub keys only, which we do not need to check, so return the first one + return tx_pub_key; } bool wallet2::export_key_images(const std::string &filename) const @@ -11309,6 +11563,17 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag } PERF_TIMER_STOP(import_key_images_C); + // accumulate outputs before the updated data + for(size_t i = 0; i < offset; ++i) + { + const transfer_details &td = m_transfers[i]; + uint64_t amount = td.amount(); + if (td.m_spent) + spent += amount; + else + unspent += amount; + } + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { @@ -11472,32 +11737,52 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag PERF_TIMER_STOP(import_key_images_G); } - return m_transfers[signed_key_images.size() - 1].m_block_height; + // this can be 0 if we do not know the height + return m_transfers[signed_key_images.size() + offset - 1].m_block_height; } -bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) +bool wallet2::import_key_images(std::vector<crypto::key_image> key_images, size_t offset, boost::optional<std::unordered_set<size_t>> selected_transfers) { - if (key_images.size() > m_transfers.size()) + if (key_images.size() + offset > m_transfers.size()) { LOG_PRINT_L1("More key images returned that we know outputs for"); return false; } - for (size_t i = 0; i < key_images.size(); ++i) + for (size_t ki_idx = 0; ki_idx < key_images.size(); ++ki_idx) { - transfer_details &td = m_transfers[i]; - if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i]) - LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); - td.m_key_image = key_images[i]; - m_key_images[m_transfers[i].m_key_image] = i; + const size_t transfer_idx = ki_idx + offset; + if (selected_transfers && selected_transfers.get().find(transfer_idx) == selected_transfers.get().end()) + continue; + + transfer_details &td = m_transfers[transfer_idx]; + if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[ki_idx]) + LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << ki_idx << ": trusting imported one"); + td.m_key_image = key_images[ki_idx]; + m_key_images[td.m_key_image] = transfer_idx; td.m_key_image_known = true; td.m_key_image_request = false; td.m_key_image_partial = false; - m_pub_keys[m_transfers[i].get_public_key()] = i; + m_pub_keys[td.get_public_key()] = transfer_idx; } return true; } +bool wallet2::import_key_images(signed_tx_set & signed_tx, size_t offset, bool only_selected_transfers) +{ + std::unordered_set<size_t> selected_transfers; + if (only_selected_transfers) + { + for (const pending_tx & ptx : signed_tx.ptx) + { + for (const size_t s: ptx.selected_transfers) + selected_transfers.insert(s); + } + } + + return import_key_images(signed_tx.key_images, offset, only_selected_transfers ? boost::make_optional(selected_transfers) : boost::none); +} + wallet2::payment_container wallet2::export_payments() const { payment_container payments; @@ -12478,22 +12763,30 @@ wallet_device_callback * wallet2::get_device_callback() } return m_device_callback.get(); }//---------------------------------------------------------------------------------------------------- -void wallet2::on_button_request() +void wallet2::on_device_button_request(uint64_t code) { - if (0 != m_callback) - m_callback->on_button_request(); + if (nullptr != m_callback) + m_callback->on_device_button_request(code); } //---------------------------------------------------------------------------------------------------- -void wallet2::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> wallet2::on_device_pin_request() { - if (0 != m_callback) - m_callback->on_pin_request(pin); + if (nullptr != m_callback) + return m_callback->on_device_pin_request(); + return boost::none; } //---------------------------------------------------------------------------------------------------- -void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device) { - if (0 != m_callback) - m_callback->on_passphrase_request(on_device, passphrase); + if (nullptr != m_callback) + return m_callback->on_device_passphrase_request(on_device); + return boost::none; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_device_progress(const hw::device_progress& event) +{ + if (nullptr != m_callback) + m_callback->on_device_progress(event); } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_rpc_status(const std::string &s) const diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 1911a8e96..28ebd6704 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -62,6 +62,7 @@ #include "common/password.h" #include "node_rpc_proxy.h" #include "message_store.h" +#include "wallet_light_rpc.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2" @@ -103,9 +104,10 @@ namespace tools virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {} // Device callbacks - virtual void on_button_request() {} - virtual void on_pin_request(epee::wipeable_string & pin) {} - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual void on_device_button_request(uint64_t code) {} + virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; } + virtual void on_device_progress(const hw::device_progress& event) {}; // Common callbacks virtual void on_pool_tx_removed(const crypto::hash &txid) {} virtual ~i_wallet2_callback() {} @@ -115,9 +117,10 @@ namespace tools { public: wallet_device_callback(wallet2 * wallet): wallet(wallet) {}; - void on_button_request() override; - void on_pin_request(epee::wipeable_string & pin) override; - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override; + void on_button_request(uint64_t code=0) override; + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; + void on_progress(const hw::device_progress& event) override; private: wallet2 * wallet; }; @@ -372,7 +375,7 @@ namespace tools std::vector<uint8_t> extra; uint64_t unlock_time; bool use_rct; - bool use_bulletproofs; + rct::RCTConfig rct_config; std::vector<cryptonote::tx_destination_entry> dests; // original setup, does not include change uint32_t subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> subaddr_indices; // set of address indices used as inputs in this transfer @@ -385,7 +388,7 @@ namespace tools FIELD(extra) FIELD(unlock_time) FIELD(use_rct) - FIELD(use_bulletproofs) + FIELD(rct_config) FIELD(dests) FIELD(subaddr_account) FIELD(subaddr_indices) @@ -678,7 +681,9 @@ namespace tools bool deinit(); bool init(std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, + boost::asio::ip::tcp::endpoint proxy = {}, + uint64_t upper_transaction_weight_limit = 0, bool trusted_daemon = true, epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect, const std::pair<std::string, std::string> &private_key_and_certificate_path = {}, @@ -797,6 +802,7 @@ namespace tools std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const; void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); @@ -1005,12 +1011,16 @@ namespace tools const std::string & device_derivation_path() const { return m_device_derivation_path; } void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } - bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; + bool get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); + bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys); void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); + void check_tx_key_helper(const cryptonote::transaction &tx, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received) const; std::string get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message); + std::string get_tx_proof(const cryptonote::transaction &tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) const; bool check_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received, bool &in_pool, uint64_t &confirmations); + bool check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const; std::string get_spend_proof(const crypto::hash &txid, const std::string &message); bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str); @@ -1128,7 +1138,8 @@ namespace tools std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); - bool import_key_images(std::vector<crypto::key_image> key_images); + bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); + bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; void update_pool_state(bool refreshed = false); @@ -1168,11 +1179,11 @@ namespace tools // fetch txs and store in m_payments void light_wallet_get_address_txs(); // get_address_info - bool light_wallet_get_address_info(cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::response &response); + bool light_wallet_get_address_info(tools::COMMAND_RPC_GET_ADDRESS_INFO::response &response); // Login. new_address is true if address hasn't been used on lw node before. bool light_wallet_login(bool &new_address); // Send an import request to lw node. returns info about import fee, address and payment_id - bool light_wallet_import_wallet_request(cryptonote::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response); + bool light_wallet_import_wallet_request(tools::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response); // get random outputs from light wallet server void light_wallet_get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); // Parse rct string @@ -1270,7 +1281,7 @@ namespace tools bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); - void detach_blockchain(uint64_t height); + void detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; bool clear(); void clear_soft(bool keep_key_images=false); @@ -1338,9 +1349,10 @@ namespace tools void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); wallet_device_callback * get_device_callback(); - void on_button_request(); - void on_pin_request(epee::wipeable_string & pin); - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + void on_device_button_request(uint64_t code); + boost::optional<epee::wipeable_string> on_device_pin_request(); + boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); + void on_device_progress(const hw::device_progress& event); std::string get_rpc_status(const std::string &s) const; void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const; @@ -1482,7 +1494,7 @@ BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) -BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) +BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 4) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1851,11 +1863,27 @@ namespace boost a & x.subaddr_account; a & x.subaddr_indices; if (ver < 2) + { + if (!typename Archive::is_saving()) + x.rct_config = { rct::RangeProofBorromean, 0 }; return; + } a & x.selected_transfers; if (ver < 3) + { + if (!typename Archive::is_saving()) + x.rct_config = { rct::RangeProofBorromean, 0 }; return; - a & x.use_bulletproofs; + } + if (ver < 4) + { + bool use_bulletproofs = x.rct_config.range_proof_type != rct::RangeProofBorromean; + a & use_bulletproofs; + if (!typename Archive::is_saving()) + x.rct_config = { use_bulletproofs ? rct::RangeProofBulletproof : rct::RangeProofBorromean, 0 }; + return; + } + a & x.rct_config; } template <class Archive> diff --git a/src/wallet/wallet_light_rpc.h b/src/wallet/wallet_light_rpc.h new file mode 100644 index 000000000..1d35cec33 --- /dev/null +++ b/src/wallet/wallet_light_rpc.h @@ -0,0 +1,320 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once +#include "cryptonote_basic/cryptonote_basic.h" +#include "crypto/hash.h" + +namespace tools +{ + //----------------------------------------------- + struct COMMAND_RPC_GET_ADDRESS_TXS + { + struct request_t + { + std::string address; + std::string view_key; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct spent_output { + uint64_t amount; + std::string key_image; + std::string tx_pub_key; + uint64_t out_index; + uint32_t mixin; + + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(key_image) + KV_SERIALIZE(tx_pub_key) + KV_SERIALIZE(out_index) + KV_SERIALIZE(mixin) + END_KV_SERIALIZE_MAP() + }; + + struct transaction + { + uint64_t id; + std::string hash; + uint64_t timestamp; + uint64_t total_received; + uint64_t total_sent; + uint64_t unlock_time; + uint64_t height; + std::list<spent_output> spent_outputs; + std::string payment_id; + bool coinbase; + bool mempool; + uint32_t mixin; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(hash) + KV_SERIALIZE(timestamp) + KV_SERIALIZE(total_received) + KV_SERIALIZE(total_sent) + KV_SERIALIZE(unlock_time) + KV_SERIALIZE(height) + KV_SERIALIZE(spent_outputs) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(coinbase) + KV_SERIALIZE(mempool) + KV_SERIALIZE(mixin) + END_KV_SERIALIZE_MAP() + }; + + + struct response_t + { + //std::list<std::string> txs_as_json; + uint64_t total_received; + uint64_t total_received_unlocked = 0; // OpenMonero only + uint64_t scanned_height; + std::vector<transaction> transactions; + uint64_t blockchain_height; + uint64_t scanned_block_height; + std::string status; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(total_received) + KV_SERIALIZE(total_received_unlocked) + KV_SERIALIZE(scanned_height) + KV_SERIALIZE(transactions) + KV_SERIALIZE(blockchain_height) + KV_SERIALIZE(scanned_block_height) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + //----------------------------------------------- + struct COMMAND_RPC_GET_ADDRESS_INFO + { + struct request_t + { + std::string address; + std::string view_key; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct spent_output + { + uint64_t amount; + std::string key_image; + std::string tx_pub_key; + uint64_t out_index; + uint32_t mixin; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(key_image) + KV_SERIALIZE(tx_pub_key) + KV_SERIALIZE(out_index) + KV_SERIALIZE(mixin) + END_KV_SERIALIZE_MAP() + }; + + struct response_t + { + uint64_t locked_funds; + uint64_t total_received; + uint64_t total_sent; + uint64_t scanned_height; + uint64_t scanned_block_height; + uint64_t start_height; + uint64_t transaction_height; + uint64_t blockchain_height; + std::list<spent_output> spent_outputs; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(locked_funds) + KV_SERIALIZE(total_received) + KV_SERIALIZE(total_sent) + KV_SERIALIZE(scanned_height) + KV_SERIALIZE(scanned_block_height) + KV_SERIALIZE(start_height) + KV_SERIALIZE(transaction_height) + KV_SERIALIZE(blockchain_height) + KV_SERIALIZE(spent_outputs) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + //----------------------------------------------- + struct COMMAND_RPC_GET_UNSPENT_OUTS + { + struct request_t + { + std::string amount; + std::string address; + std::string view_key; + // OpenMonero specific + uint64_t mixin; + bool use_dust; + std::string dust_threshold; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + KV_SERIALIZE(mixin) + KV_SERIALIZE(use_dust) + KV_SERIALIZE(dust_threshold) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + + struct output { + uint64_t amount; + std::string public_key; + uint64_t index; + uint64_t global_index; + std::string rct; + std::string tx_hash; + std::string tx_pub_key; + std::string tx_prefix_hash; + std::vector<std::string> spend_key_images; + uint64_t timestamp; + uint64_t height; + + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(public_key) + KV_SERIALIZE(index) + KV_SERIALIZE(global_index) + KV_SERIALIZE(rct) + KV_SERIALIZE(tx_hash) + KV_SERIALIZE(tx_pub_key) + KV_SERIALIZE(tx_prefix_hash) + KV_SERIALIZE(spend_key_images) + KV_SERIALIZE(timestamp) + KV_SERIALIZE(height) + END_KV_SERIALIZE_MAP() + }; + + struct response_t + { + uint64_t amount; + std::list<output> outputs; + uint64_t per_kb_fee; + std::string status; + std::string reason; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(outputs) + KV_SERIALIZE(per_kb_fee) + KV_SERIALIZE(status) + KV_SERIALIZE(reason) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- + struct COMMAND_RPC_LOGIN + { + struct request_t + { + std::string address; + std::string view_key; + bool create_account; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + KV_SERIALIZE(create_account) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string status; + std::string reason; + bool new_address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(reason) + KV_SERIALIZE(new_address) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- + struct COMMAND_RPC_IMPORT_WALLET_REQUEST + { + struct request_t + { + std::string address; + std::string view_key; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string payment_id; + uint64_t import_fee; + bool new_request; + bool request_fulfilled; + std::string payment_address; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(payment_id) + KV_SERIALIZE(import_fee) + KV_SERIALIZE(new_request) + KV_SERIALIZE(request_fulfilled) + KV_SERIALIZE(payment_address) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- +} diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 7040597df..18b2de33f 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -56,6 +56,8 @@ using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc" +#define DEFAULT_AUTO_REFRESH_PERIOD 20 // seconds + namespace { const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; @@ -124,13 +126,18 @@ namespace tools { m_stop = false; m_net_server.add_idle_handler([this](){ + if (m_auto_refresh_period == 0) // disabled + return true; + if (boost::posix_time::microsec_clock::universal_time() < m_last_auto_refresh_time + boost::posix_time::seconds(m_auto_refresh_period)) + return true; try { if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } + m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time(); return true; - }, 20000); + }, 1000); m_net_server.add_idle_handler([this](){ if (m_stop.load(std::memory_order_relaxed)) { @@ -263,6 +270,9 @@ namespace tools std::vector<std::vector<uint8_t>> allowed_fingerprints{ rpc_ssl_allowed_fingerprints.size() }; std::transform(rpc_ssl_allowed_fingerprints.begin(), rpc_ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); + m_auto_refresh_period = DEFAULT_AUTO_REFRESH_PERIOD; + m_last_auto_refresh_time = boost::posix_time::min_date_time; + m_net_server.set_threads_prefix("RPC"); auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( @@ -319,6 +329,8 @@ namespace tools entry.type = "out"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; + for (uint32_t i: pd.m_subaddr_indices) + entry.subaddr_indices.push_back({pd.m_subaddr_account, i}); entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } @@ -339,6 +351,8 @@ namespace tools entry.note = m_wallet->get_tx_note(txid); entry.type = is_failed ? "failed" : "pending"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; + for (uint32_t i: pd.m_subaddr_indices) + entry.subaddr_indices.push_back({pd.m_subaddr_account, i}); entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } @@ -2830,6 +2844,28 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try + { + m_auto_refresh_period = req.enable ? req.period ? req.period : DEFAULT_AUTO_REFRESH_PERIOD : 0; + MINFO("Auto refresh now " << (m_auto_refresh_period ? std::to_string(m_auto_refresh_period) + " seconds" : std::string("disabled"))); + return true; + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2932,7 +2968,7 @@ namespace tools er.message = "Invalid filename"; return false; } - std::string wallet_file = m_wallet_dir + "/" + req.filename; + std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); { std::vector<std::string> languages; crypto::ElectrumWords::get_language_list(languages); @@ -3217,12 +3253,6 @@ namespace tools } // early check for mandatory fields - if (req.filename.empty()) - { - er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; - er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; - return false; - } if (req.viewkey.empty()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -3251,7 +3281,7 @@ namespace tools er.message = "Invalid filename"; return false; } - std::string wallet_file = m_wallet_dir + "/" + req.filename; + std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); // check if wallet file already exists if (!wallet_file.empty()) { @@ -3361,7 +3391,8 @@ namespace tools { try { - m_wallet->store(); + if (!wallet_file.empty()) + m_wallet->store(); } catch (const std::exception &e) { @@ -3385,12 +3416,6 @@ namespace tools } // early check for mandatory fields - if (req.filename.empty()) - { - er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; - er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; - return false; - } if (req.seed.empty()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -3413,7 +3438,7 @@ namespace tools er.message = "Invalid filename"; return false; } - std::string wallet_file = m_wallet_dir + "/" + req.filename; + std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); // check if wallet file already exists if (!wallet_file.empty()) { @@ -3529,7 +3554,7 @@ namespace tools er.message = "Failed to encode seed"; return false; } - res.seed = electrum_words.data(); + res.seed = std::string(electrum_words.data(), electrum_words.size()); if (!wal) { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index affaf10f7..2b52275b8 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -129,6 +129,7 @@ namespace tools MAP_JON_RPC_WE("add_address_book", on_add_address_book, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) + MAP_JON_RPC_WE("auto_refresh", on_auto_refresh, wallet_rpc::COMMAND_RPC_AUTO_REFRESH) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) @@ -210,6 +211,7 @@ namespace tools bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); @@ -256,5 +258,7 @@ namespace tools std::atomic<bool> m_stop; bool m_restricted; const boost::program_options::variables_map *m_vm; + uint32_t m_auto_refresh_period; + boost::posix_time::ptime m_last_auto_refresh_time; }; } diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index b0e8bed93..df4370949 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -1359,6 +1359,7 @@ namespace wallet_rpc std::string type; uint64_t unlock_time; cryptonote::subaddress_index subaddr_index; + std::vector<cryptonote::subaddress_index> subaddr_indices; std::string address; bool double_spend_seen; uint64_t confirmations; @@ -1376,6 +1377,7 @@ namespace wallet_rpc KV_SERIALIZE(type); KV_SERIALIZE(unlock_time) KV_SERIALIZE(subaddr_index); + KV_SERIALIZE(subaddr_indices); KV_SERIALIZE(address); KV_SERIALIZE(double_spend_seen) KV_SERIALIZE_OPT(confirmations, (uint64_t)0) @@ -1930,6 +1932,28 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_AUTO_REFRESH + { + struct request_t + { + bool enable; + uint32_t period; // seconds + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(enable, true) + KV_SERIALIZE_OPT(period, (uint32_t)0) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_START_MINING { struct request_t diff --git a/tests/README.md b/tests/README.md index 001ab6154..053dd2244 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,7 +18,7 @@ Tests are located in `tests/core_tests/`, and follow a straightforward naming co To run only Monero's core tests (after building): ``` -cd build/debug/tests/core +cd build/debug/tests/core_tests ctest ``` @@ -103,6 +103,8 @@ cd build/debug/tests/performance_tests ./performance_tests ``` +The path may be build/Linux/master/debug (adapt as necessary for your platform). + If the `performance_tests` binary does not exist, try running `make` in the `build/debug/tests/performance_tests` directory. To run the same tests on a release build, replace `debug` with `release`. diff --git a/tests/block_weight/CMakeLists.txt b/tests/block_weight/CMakeLists.txt index b0d716ea0..a59c9c1fc 100644 --- a/tests/block_weight/CMakeLists.txt +++ b/tests/block_weight/CMakeLists.txt @@ -42,4 +42,4 @@ target_link_libraries(block_weight add_test( NAME block_weight - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight) + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight) diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp index 57fcb497e..1e8974b78 100644 --- a/tests/block_weight/block_weight.cpp +++ b/tests/block_weight/block_weight.cpp @@ -72,6 +72,18 @@ public: virtual uint64_t height() const override { return blocks.size(); } virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight); + return ret; + } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight); + return ret; + } virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override { uint64_t h = height(); crypto::hash top = crypto::null_hash; diff --git a/tests/block_weight/block_weight.py b/tests/block_weight/block_weight.py index 06aaabb02..d6fd494e3 100755 --- a/tests/block_weight/block_weight.py +++ b/tests/block_weight/block_weight.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Simulate a maximal block attack on the Monero network # This uses the scheme proposed by ArticMine # Written by Sarang Nother diff --git a/tests/block_weight/compare.py b/tests/block_weight/compare.py index c6be05206..e058a7079 100755 --- a/tests/block_weight/compare.py +++ b/tests/block_weight/compare.py @@ -3,10 +3,17 @@ import sys import subprocess -print 'running: ', sys.argv[1] -S0 = subprocess.check_output(sys.argv[1], stderr=subprocess.STDOUT) -print 'running: ', sys.argv[2] -S1 = subprocess.check_output(sys.argv[2], stderr=subprocess.STDOUT) +if len(sys.argv) == 4: + first = [sys.argv[1], sys.argv[2]] + second = [sys.argv[3]] +else: + first = [sys.argv[1]] + second = [sys.argv[2]] + +print 'running: ', first +S0 = subprocess.check_output(first, stderr=subprocess.STDOUT) +print 'running: ', second +S1 = subprocess.check_output(second, stderr=subprocess.STDOUT) print 'comparing' if S0 != S1: sys.exit(1) diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 8bf5a9b08..f2bcb7787 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -754,7 +754,7 @@ struct get_test_options { }; //-------------------------------------------------------------------------- template<class t_test_class> -inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core **core) +inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core *core) { boost::program_options::options_description desc("Allowed options"); cryptonote::core::init_options(desc); @@ -768,8 +768,7 @@ inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cry if (!r) return false; - *core = new cryptonote::core(nullptr); - auto & c = **core; + auto & c = *core; // FIXME: make sure that vm has arg_testnet_on set to true or false if // this test needs for it to be so. @@ -825,9 +824,9 @@ inline bool replay_events_through_core_validate(std::vector<test_event_entry>& e template<class t_test_class> inline bool do_replay_events(std::vector<test_event_entry>& events) { - cryptonote::core * core; + cryptonote::core core(nullptr); bool ret = do_replay_events_get_core<t_test_class>(events, &core); - core->deinit(); + core.deinit(); return ret; } //-------------------------------------------------------------------------- diff --git a/tests/core_tests/wallet_tools.cpp b/tests/core_tests/wallet_tools.cpp index ff7ce3a34..616774d18 100644 --- a/tests/core_tests/wallet_tools.cpp +++ b/tests/core_tests/wallet_tools.cpp @@ -17,7 +17,6 @@ void wallet_accessor_test::set_account(tools::wallet2 * wallet, cryptonote::acco { wallet->clear(); wallet->m_account = account; - wallet->m_nettype = MAINNET; wallet->m_key_device_type = account.get_device().get_type(); wallet->m_account_public_address = account.get_keys().m_account_address; diff --git a/tests/difficulty/CMakeLists.txt b/tests/difficulty/CMakeLists.txt index 2ed495806..fb0dd6b9e 100644 --- a/tests/difficulty/CMakeLists.txt +++ b/tests/difficulty/CMakeLists.txt @@ -45,3 +45,6 @@ set_property(TARGET difficulty-tests add_test( NAME difficulty COMMAND difficulty-tests "${CMAKE_CURRENT_SOURCE_DIR}/data.txt") +add_test( + NAME wide_difficulty + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/wide_difficulty.py" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/gen_wide_data.py" "${CMAKE_CURRENT_BINARY_DIR}/difficulty-tests" "${CMAKE_CURRENT_BINARY_DIR}/wide_data.txt") diff --git a/tests/difficulty/difficulty.cpp b/tests/difficulty/difficulty.cpp index ee20e27e4..9985b8710 100644 --- a/tests/difficulty/difficulty.cpp +++ b/tests/difficulty/difficulty.cpp @@ -43,16 +43,62 @@ using namespace std; #define DEFAULT_TEST_DIFFICULTY_TARGET 120 +static int test_wide_difficulty(const char *filename) +{ + std::vector<uint64_t> timestamps; + std::vector<cryptonote::difficulty_type> cumulative_difficulties; + fstream data(filename, fstream::in); + data.exceptions(fstream::badbit); + data.clear(data.rdstate()); + uint64_t timestamp; + cryptonote::difficulty_type difficulty, cumulative_difficulty = 0; + size_t n = 0; + while (data >> timestamp >> difficulty) { + size_t begin, end; + if (n < DIFFICULTY_WINDOW + DIFFICULTY_LAG) { + begin = 0; + end = min(n, (size_t) DIFFICULTY_WINDOW); + } else { + end = n - DIFFICULTY_LAG; + begin = end - DIFFICULTY_WINDOW; + } + cryptonote::difficulty_type res = cryptonote::next_difficulty( + std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end), + std::vector<cryptonote::difficulty_type>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET); + if (res != difficulty) { + cerr << "Wrong wide difficulty for block " << n << endl + << "Expected: " << difficulty << endl + << "Found: " << res << endl; + return 1; + } + timestamps.push_back(timestamp); + cumulative_difficulties.push_back(cumulative_difficulty += difficulty); + ++n; + } + if (!data.eof()) { + data.clear(fstream::badbit); + } + return 0; +} + int main(int argc, char *argv[]) { - if (argc != 2) { + if (argc < 2) { cerr << "Wrong arguments" << endl; return 1; } + if (argc == 3 && strcmp(argv[1], "--wide") == 0) + { + return test_wide_difficulty(argv[2]); + } + vector<uint64_t> timestamps, cumulative_difficulties; + std::vector<cryptonote::difficulty_type> wide_cumulative_difficulties; fstream data(argv[1], fstream::in); data.exceptions(fstream::badbit); data.clear(data.rdstate()); - uint64_t timestamp, difficulty, cumulative_difficulty = 0; + uint64_t timestamp; + uint64_t difficulty, cumulative_difficulty = 0; + cryptonote::difficulty_type wide_cumulative_difficulty = 0; size_t n = 0; while (data >> timestamp >> difficulty) { size_t begin, end; @@ -63,17 +109,27 @@ int main(int argc, char *argv[]) { end = n - DIFFICULTY_LAG; begin = end - DIFFICULTY_WINDOW; } - uint64_t res = cryptonote::next_difficulty( + uint64_t res = cryptonote::next_difficulty_64( vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end), - vector<uint64_t>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET); + std::vector<uint64_t>(cumulative_difficulties.begin() + begin, cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET); if (res != difficulty) { cerr << "Wrong difficulty for block " << n << endl << "Expected: " << difficulty << endl << "Found: " << res << endl; return 1; } + cryptonote::difficulty_type wide_res = cryptonote::next_difficulty( + std::vector<uint64_t>(timestamps.begin() + begin, timestamps.begin() + end), + std::vector<cryptonote::difficulty_type>(wide_cumulative_difficulties.begin() + begin, wide_cumulative_difficulties.begin() + end), DEFAULT_TEST_DIFFICULTY_TARGET); + if (wide_res.convert_to<uint64_t>() != res) { + cerr << "Wrong wide difficulty for block " << n << endl + << "Expected: " << res << endl + << "Found: " << wide_res << endl; + return 1; + } timestamps.push_back(timestamp); cumulative_difficulties.push_back(cumulative_difficulty += difficulty); + wide_cumulative_difficulties.push_back(wide_cumulative_difficulty += difficulty); ++n; } if (!data.eof()) { diff --git a/tests/difficulty/gen_wide_data.py b/tests/difficulty/gen_wide_data.py new file mode 100644 index 000000000..64af4e208 --- /dev/null +++ b/tests/difficulty/gen_wide_data.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +from __future__ import print_function +import random + +DIFFICULTY_TARGET = 120 +DIFFICULTY_WINDOW = 720 +DIFFICULTY_LAG = 15 +DIFFICULTY_CUT = 60 + +def difficulty(): + times = [] + diffs = [] + while True: + if len(times) <= 1: + diff = 1 + else: + begin = max(len(times) - DIFFICULTY_WINDOW - DIFFICULTY_LAG, 0) + end = min(begin + DIFFICULTY_WINDOW, len(times)) + length = end - begin + assert length >= 2 + if length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT: + cut_begin = 0 + cut_end = length + else: + excess = length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + cut_begin = (excess + 1) // 2 + cut_end = length - excess // 2 + assert cut_begin + 2 <= cut_end + wnd = times[begin:end] + wnd.sort() + dtime = wnd[cut_end - 1] - wnd[cut_begin] + dtime = max(dtime, 1) + ddiff = sum(diffs[begin + cut_begin + 1:begin + cut_end]) + diff = (ddiff * DIFFICULTY_TARGET + dtime - 1) // dtime + times.append((yield diff)) + diffs.append(diff) + +random.seed(1) +time = 1000 +gen = difficulty() +diff = next(gen) +for i in range(100000): + power = 100 if i < 10000 else 100000000 if i < 500 else 1000000000000 if i < 1000 else 1000000000000000 if i < 2000 else 10000000000000000000 if i < 4000 else 1000000000000000000000000 + time += random.randint(-diff // power - 10, 3 * diff // power + 10) + print(time, diff) + diff = gen.send(time) diff --git a/tests/difficulty/wide_difficulty.py b/tests/difficulty/wide_difficulty.py new file mode 100755 index 000000000..41a2a632d --- /dev/null +++ b/tests/difficulty/wide_difficulty.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +from __future__ import print_function +import sys +import subprocess + +python = sys.argv[1] +py = sys.argv[2] +c = sys.argv[3] +data = sys.argv[4] + +first = python + " " + py + " > " + data +second = [c, '--wide', data] + +try: + print('running: ', first) + subprocess.check_call(first, shell=True) + print('running: ', second) + subprocess.check_call(second) +except: + sys.exit(1) + diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index f3f35e140..f4050c948 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -53,7 +53,7 @@ int ColdOutputsFuzzer::init() try { - wallet.init(""); + wallet.init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet.set_subaddress_lookahead(1, 1); wallet.generate("", "", spendkey, true, false); } diff --git a/tests/fuzz/cold-transaction.cpp b/tests/fuzz/cold-transaction.cpp index 83493f8a0..08117281b 100644 --- a/tests/fuzz/cold-transaction.cpp +++ b/tests/fuzz/cold-transaction.cpp @@ -54,7 +54,7 @@ int ColdTransactionFuzzer::init() try { - wallet.init(""); + wallet.init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet.set_subaddress_lookahead(1, 1); wallet.generate("", "", spendkey, true, false); } diff --git a/tests/fuzz/signature.cpp b/tests/fuzz/signature.cpp index ab8faa29f..038378ae2 100644 --- a/tests/fuzz/signature.cpp +++ b/tests/fuzz/signature.cpp @@ -54,7 +54,7 @@ int SignatureFuzzer::init() try { - wallet.init(""); + wallet.init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet.set_subaddress_lookahead(1, 1); wallet.generate("", "", spendkey, true, false); diff --git a/tests/gtest/include/gtest/internal/gtest-port-arch.h b/tests/gtest/include/gtest/internal/gtest-port-arch.h index 74ab94905..83022c82c 100644 --- a/tests/gtest/include/gtest/internal/gtest-port-arch.h +++ b/tests/gtest/include/gtest/internal/gtest-port-arch.h @@ -88,6 +88,8 @@ # define GTEST_OS_OPENBSD 1 #elif defined __QNX__ # define GTEST_OS_QNX 1 +#elif defined(__NetBSD__) +# define GTEST_OS_NETBSD 1 #endif // __CYGWIN__ #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ diff --git a/tests/gtest/include/gtest/internal/gtest-port.h b/tests/gtest/include/gtest/internal/gtest-port.h index cb0639b4b..2896dfe64 100644 --- a/tests/gtest/include/gtest/internal/gtest-port.h +++ b/tests/gtest/include/gtest/internal/gtest-port.h @@ -790,7 +790,8 @@ using ::std::tuple_size; (GTEST_OS_MAC && !GTEST_OS_IOS) || \ (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \ - GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD) + GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD || \ + GTEST_OS_NETBSD) # define GTEST_HAS_DEATH_TEST 1 #endif diff --git a/tests/hash-target.cpp b/tests/hash-target.cpp index 70368ce24..12acc5a67 100644 --- a/tests/hash-target.cpp +++ b/tests/hash-target.cpp @@ -40,7 +40,7 @@ using cryptonote::check_hash; int main(int argc, char *argv[]) { crypto::hash h; - for (uint64_t diff = 1;; diff += 1 + (diff >> 8)) { + for (cryptonote::difficulty_type diff = 1;; diff += 1 + (diff >> 8)) { for (uint16_t b = 0; b < 256; b++) { memset(&h, b, sizeof(crypto::hash)); if (check_hash(h, diff) != (b == 0 || diff <= 255 / b)) { @@ -50,7 +50,7 @@ int main(int argc, char *argv[]) { memset(&h, 0, sizeof(crypto::hash)); ((char *) &h)[31] = b; if (check_hash(h, diff) != (diff <= 255 / b)) { - return 1; + return 2; } } } @@ -58,11 +58,11 @@ int main(int argc, char *argv[]) { uint64_t val = 0; for (int i = 31; i >= 0; i--) { val = val * 256 + 255; - ((char *) &h)[i] = static_cast<char>(val / diff); - val %= diff; + ((char *) &h)[i] = static_cast<char>(static_cast<uint64_t>(val / diff)); + val %= diff.convert_to<uint64_t>(); } if (check_hash(h, diff) != true) { - return 1; + return 3; } if (diff > 1) { for (int i = 0;; i++) { @@ -74,7 +74,7 @@ int main(int argc, char *argv[]) { } } if (check_hash(h, diff) != false) { - return 1; + return 4; } } } diff --git a/tests/performance_tests/CMakeLists.txt b/tests/performance_tests/CMakeLists.txt index aa424da80..b36df10dc 100644 --- a/tests/performance_tests/CMakeLists.txt +++ b/tests/performance_tests/CMakeLists.txt @@ -31,6 +31,7 @@ set(performance_tests_sources set(performance_tests_headers check_tx_signature.h + check_hash.h cn_slow_hash.h construct_tx.h derive_public_key.h diff --git a/tests/performance_tests/check_hash.h b/tests/performance_tests/check_hash.h new file mode 100644 index 000000000..d24001903 --- /dev/null +++ b/tests/performance_tests/check_hash.h @@ -0,0 +1,66 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "string_tools.h" +#include "cryptonote_basic/difficulty.h" + +template<uint64_t hash_target_high, uint64_t hash_target_low, uint64_t difficulty_high, uint64_t difficulty_low> +class test_check_hash +{ +public: + static const size_t loop_count = 100000; + + bool init() + { + cryptonote::difficulty_type hash_target = hash_target_high; + hash_target = (hash_target << 64) | hash_target_low; + difficulty = difficulty_high; + difficulty = (difficulty << 64) | difficulty_low; + boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; + ((uint64_t*)&hash)[0] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + hash_value >>= 64; + ((uint64_t*)&hash)[1] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + hash_value >>= 64; + ((uint64_t*)&hash)[2] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + hash_value >>= 64; + ((uint64_t*)&hash)[3] = (hash_value << 64 >> 64).convert_to<uint64_t>(); + return true; + } + + bool test() + { + cryptonote::check_hash_128(hash, difficulty); + return true; + } + +private: + crypto::hash hash; + cryptonote::difficulty_type difficulty; +}; diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index e6558a364..c32e0df20 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -38,6 +38,7 @@ // tests #include "construct_tx.h" #include "check_tx_signature.h" +#include "check_hash.h" #include "cn_slow_hash.h" #include "derive_public_key.h" #include "derive_secret_key.h" @@ -181,6 +182,14 @@ int main(int argc, char** argv) TEST_PERFORMANCE4(filter, p, test_check_tx_signature_aggregated_bulletproofs, 2, 2, 56, 16); TEST_PERFORMANCE4(filter, p, test_check_tx_signature_aggregated_bulletproofs, 10, 2, 56, 16); + TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 1, 0, 1); + TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 0xffffffffffffffff, 0, 0xffffffffffffffff); + TEST_PERFORMANCE4(filter, p, test_check_hash, 0, 0xffffffffffffffff, 0, 1); + TEST_PERFORMANCE4(filter, p, test_check_hash, 1, 0, 1, 0); + TEST_PERFORMANCE4(filter, p, test_check_hash, 1, 0, 0, 1); + TEST_PERFORMANCE4(filter, p, test_check_hash, 0xffffffffffffffff, 0xffffffffffffffff, 0, 1); + TEST_PERFORMANCE4(filter, p, test_check_hash, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff); + TEST_PERFORMANCE0(filter, p, test_is_out_to_acc); TEST_PERFORMANCE0(filter, p, test_is_out_to_acc_precomp); TEST_PERFORMANCE0(filter, p, test_generate_key_image_helper); diff --git a/tests/trezor/CMakeLists.txt b/tests/trezor/CMakeLists.txt index 67c2f8438..15ed5668d 100644 --- a/tests/trezor/CMakeLists.txt +++ b/tests/trezor/CMakeLists.txt @@ -27,11 +27,15 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set(trezor_tests_sources + tools.cpp + daemon.cpp trezor_tests.cpp ../core_tests/chaingen.cpp ../core_tests/wallet_tools.cpp) set(trezor_tests_headers + tools.h + daemon.h trezor_tests.h ../core_tests/chaingen.h ../core_tests/wallet_tools.h) @@ -50,6 +54,15 @@ target_link_libraries(trezor_tests device device_trezor wallet + wallet_api + rpc + cryptonote_protocol + daemon_rpc_server + ${Boost_CHRONO_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${ZMQ_LIB} ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) diff --git a/tests/trezor/daemon.cpp b/tests/trezor/daemon.cpp new file mode 100644 index 000000000..5e987793a --- /dev/null +++ b/tests/trezor/daemon.cpp @@ -0,0 +1,368 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "daemon.h" +#include <common/command_line.h> + +using namespace std; +using namespace daemonize; +namespace po = boost::program_options; + +bool mock_rpc_daemon::on_send_raw_tx_2(const cryptonote::COMMAND_RPC_SEND_RAW_TX::request& req, cryptonote::COMMAND_RPC_SEND_RAW_TX::response& res, const cryptonote::core_rpc_server::connection_context *ctx) +{ + cryptonote::COMMAND_RPC_SEND_RAW_TX::request req2(req); + req2.do_not_relay = true; // Do not relay in test setup, only one daemon running. + return cryptonote::core_rpc_server::on_send_raw_tx(req2, res, ctx); +} + +void mock_daemon::init_options(boost::program_options::options_description & option_spec) +{ + cryptonote::core::init_options(option_spec); + t_node_server::init_options(option_spec); + cryptonote::core_rpc_server::init_options(option_spec); + + command_line::add_arg(option_spec, daemon_args::arg_zmq_rpc_bind_ip); + command_line::add_arg(option_spec, daemon_args::arg_zmq_rpc_bind_port); +} + +void mock_daemon::default_options(boost::program_options::variables_map & vm) +{ + std::vector<std::string> exclusive_nodes{"127.0.0.1:65525"}; + tools::options::set_option(vm, nodetool::arg_p2p_add_exclusive_node, po::variable_value(exclusive_nodes, false)); + + tools::options::set_option(vm, nodetool::arg_p2p_bind_ip, po::variable_value(std::string("127.0.0.1"), false)); + tools::options::set_option(vm, nodetool::arg_no_igd, po::variable_value(true, false)); + tools::options::set_option(vm, cryptonote::arg_offline, po::variable_value(true, false)); + tools::options::set_option(vm, "disable-dns-checkpoints", po::variable_value(true, false)); + + const char *test_mainnet = getenv("TEST_MAINNET"); + if (!test_mainnet || atoi(test_mainnet) == 0) + { + tools::options::set_option(vm, cryptonote::arg_testnet_on, po::variable_value(true, false)); + } + + // By default pick non-standard ports to avoid confusion with possibly locally running daemons (mainnet/testnet) + const char *test_p2p_port = getenv("TEST_P2P_PORT"); + auto p2p_port = std::string(test_p2p_port && strlen(test_p2p_port) > 0 ? test_p2p_port : "61340"); + tools::options::set_option(vm, nodetool::arg_p2p_bind_port, po::variable_value(p2p_port, false)); + + const char *test_rpc_port = getenv("TEST_RPC_PORT"); + auto rpc_port = std::string(test_rpc_port && strlen(test_rpc_port) > 0 ? test_rpc_port : "61341"); + tools::options::set_option(vm, cryptonote::core_rpc_server::arg_rpc_bind_port, po::variable_value(rpc_port, false)); + + const char *test_zmq_port = getenv("TEST_ZMQ_PORT"); + auto zmq_port = std::string(test_zmq_port && strlen(test_zmq_port) > 0 ? test_zmq_port : "61342"); + tools::options::set_option(vm, daemon_args::arg_zmq_rpc_bind_port, po::variable_value(zmq_port, false)); + + po::notify(vm); +} + +void mock_daemon::set_ports(boost::program_options::variables_map & vm, unsigned initial_port) +{ + CHECK_AND_ASSERT_THROW_MES(initial_port < 65535-2, "Invalid port number"); + tools::options::set_option(vm, nodetool::arg_p2p_bind_port, po::variable_value(std::to_string(initial_port), false)); + tools::options::set_option(vm, cryptonote::core_rpc_server::arg_rpc_bind_port, po::variable_value(std::to_string(initial_port + 1), false)); + tools::options::set_option(vm, daemon_args::arg_zmq_rpc_bind_port, po::variable_value(std::to_string(initial_port + 2), false)); + po::notify(vm); +} + +void mock_daemon::load_params(boost::program_options::variables_map const & vm) +{ + m_p2p_bind_port = command_line::get_arg(vm, nodetool::arg_p2p_bind_port); + m_rpc_bind_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + m_zmq_bind_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port); + m_network_type = command_line::get_arg(vm, cryptonote::arg_testnet_on) ? cryptonote::TESTNET : cryptonote::MAINNET; +} + +mock_daemon::~mock_daemon() +{ + if (!m_terminated) + { + try + { + stop(); + } + catch (...) + { + MERROR("Failed to stop"); + } + } + + if (!m_deinitalized) + { + deinit(); + } +} + +void mock_daemon::init() +{ + m_deinitalized = false; + const auto main_rpc_port = command_line::get_arg(m_vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + m_rpc_server.nettype(m_network_type); + + CHECK_AND_ASSERT_THROW_MES(m_protocol.init(m_vm), "Failed to initialize cryptonote protocol."); + CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port), "Failed to initialize RPC server."); + + if (m_start_p2p) + CHECK_AND_ASSERT_THROW_MES(m_server.init(m_vm), "Failed to initialize p2p server."); + + if(m_http_client.is_connected()) + m_http_client.disconnect(); + + CHECK_AND_ASSERT_THROW_MES(m_http_client.set_server(rpc_addr(), boost::none), "RPC client init fail"); +} + +void mock_daemon::deinit() +{ + try + { + m_rpc_server.deinit(); + } + catch (...) + { + MERROR("Failed to deinitialize RPC server..."); + } + + if (m_start_p2p) + { + try + { + m_server.deinit(); + } + catch (...) + { + MERROR("Failed to deinitialize p2p..."); + } + } + + try + { + m_protocol.deinit(); + m_protocol.set_p2p_endpoint(nullptr); + } + catch (...) + { + MERROR("Failed to stop cryptonote protocol!"); + } + + m_deinitalized = true; +} + +void mock_daemon::init_and_run() +{ + init(); + run(); +} + +void mock_daemon::stop_and_deinit() +{ + stop(); + deinit(); +} + +void mock_daemon::try_init_and_run(boost::optional<unsigned> initial_port) +{ + const unsigned max_attempts = 3; + for(unsigned attempts=0; attempts < max_attempts; ++attempts) + { + if (initial_port) + { + set_ports(m_vm, initial_port.get()); + load_params(m_vm); + MDEBUG("Ports changed, RPC: " << rpc_addr()); + } + + try + { + init_and_run(); + return; + } + catch(const std::exception &e) + { + MWARNING("Could not init and start, attempt: " << attempts << ", reason: " << e.what()); + if (attempts + 1 >= max_attempts) + { + throw; + } + } + } +} + +void mock_daemon::run() +{ + m_run_thread = boost::thread(boost::bind(&mock_daemon::run_main, this)); +} + +bool mock_daemon::run_main() +{ + CHECK_AND_ASSERT_THROW_MES(!m_terminated, "Can't run stopped daemon"); + CHECK_AND_ASSERT_THROW_MES(!m_start_zmq || m_start_p2p, "ZMQ requires P2P"); + boost::thread stop_thread = boost::thread([this] { + while (!this->m_stopped) + epee::misc_utils::sleep_no_w(100); + this->stop_p2p(); + }); + + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + m_stopped = true; + stop_thread.join(); + }); + + try + { + CHECK_AND_ASSERT_THROW_MES(m_rpc_server.run(2, false), "Failed to start RPC"); + cryptonote::rpc::DaemonHandler rpc_daemon_handler(*m_core, m_server); + cryptonote::rpc::ZmqServer zmq_server(rpc_daemon_handler); + + if (m_start_zmq) + { + if (!zmq_server.addTCPSocket("127.0.0.1", m_zmq_bind_port)) + { + MERROR("Failed to add TCP Socket (127.0.0.1:" << m_zmq_bind_port << ") to ZMQ RPC Server"); + + stop_rpc(); + return false; + } + + MINFO("Starting ZMQ server..."); + zmq_server.run(); + + MINFO("ZMQ server started at 127.0.0.1: " << m_zmq_bind_port); + } + + if (m_start_p2p) + { + m_server.run(); // blocks until p2p goes down + } + else + { + while (!this->m_stopped) + epee::misc_utils::sleep_no_w(100); + } + + if (m_start_zmq) + zmq_server.stop(); + + stop_rpc(); + return true; + } + catch (std::exception const & ex) + { + MFATAL("Uncaught exception! " << ex.what()); + return false; + } + catch (...) + { + MFATAL("Uncaught exception!"); + return false; + } +} + +void mock_daemon::stop() +{ + CHECK_AND_ASSERT_THROW_MES(!m_terminated, "Can't stop stopped daemon"); + m_stopped = true; + m_terminated = true; + m_run_thread.join(); +} + +void mock_daemon::stop_rpc() +{ + m_rpc_server.send_stop_signal(); + m_rpc_server.timed_wait_server_stop(5000); +} + +void mock_daemon::stop_p2p() +{ + if (m_start_p2p) + m_server.send_stop_signal(); +} + +void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address) +{ + bool blocks_mined = false; + const uint64_t start_height = get_height(); + const auto mining_timeout = std::chrono::seconds(30); + MDEBUG("Current height before mining: " << start_height); + + start_mining(miner_address); + auto mining_started = std::chrono::system_clock::now(); + + while(true) { + epee::misc_utils::sleep_no_w(100); + const uint64_t cur_height = get_height(); + + if (cur_height - start_height >= num_blocks) + { + MDEBUG("Cur blocks: " << cur_height << " start: " << start_height); + blocks_mined = true; + break; + } + + auto current_time = std::chrono::system_clock::now(); + if (mining_timeout < current_time - mining_started) + { + break; + } + } + + stop_mining(); + CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit"); +} + +constexpr const std::chrono::seconds mock_daemon::rpc_timeout; + +void mock_daemon::start_mining(const std::string &miner_address, uint64_t threads_count, bool do_background_mining, bool ignore_battery) +{ + cryptonote::COMMAND_RPC_START_MINING::request req; + req.miner_address = miner_address; + req.threads_count = threads_count; + req.do_background_mining = do_background_mining; + req.ignore_battery = ignore_battery; + + cryptonote::COMMAND_RPC_START_MINING::response resp; + bool r = epee::net_utils::invoke_http_json("/start_mining", req, resp, m_http_client, rpc_timeout); + CHECK_AND_ASSERT_THROW_MES(r, "RPC error - start mining"); + CHECK_AND_ASSERT_THROW_MES(resp.status != CORE_RPC_STATUS_BUSY, "Daemon busy"); + CHECK_AND_ASSERT_THROW_MES(resp.status == CORE_RPC_STATUS_OK, "Daemon response invalid: " << resp.status); +} + +void mock_daemon::stop_mining() +{ + cryptonote::COMMAND_RPC_STOP_MINING::request req; + cryptonote::COMMAND_RPC_STOP_MINING::response resp; + bool r = epee::net_utils::invoke_http_json("/stop_mining", req, resp, m_http_client, rpc_timeout); + CHECK_AND_ASSERT_THROW_MES(r, "RPC error - stop mining"); + CHECK_AND_ASSERT_THROW_MES(resp.status != CORE_RPC_STATUS_BUSY, "Daemon busy"); + CHECK_AND_ASSERT_THROW_MES(resp.status == CORE_RPC_STATUS_OK, "Daemon response invalid: " << resp.status); +} + +uint64_t mock_daemon::get_height() +{ + return m_core->get_blockchain_storage().get_current_blockchain_height(); +} diff --git a/tests/trezor/daemon.h b/tests/trezor/daemon.h new file mode 100644 index 000000000..046b09a5d --- /dev/null +++ b/tests/trezor/daemon.h @@ -0,0 +1,154 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "misc_log_ex.h" +#include "daemon/daemon.h" +#include "rpc/daemon_handler.h" +#include "rpc/zmq_server.h" +#include "common/password.h" +#include "common/util.h" +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "daemon/protocol.h" +#include "daemon/rpc.h" +#include "daemon/command_server.h" +#include "daemon/command_server.h" +#include "daemon/command_line_args.h" +#include "version.h" +#include "tools.h" + + +class mock_rpc_daemon : public cryptonote::core_rpc_server { +public: + mock_rpc_daemon( + cryptonote::core& cr + , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p + ): cryptonote::core_rpc_server(cr, p2p) {} + + static void init_options(boost::program_options::options_description& desc){ cryptonote::core_rpc_server::init_options(desc); } + cryptonote::network_type nettype() const { return m_network_type; } + void nettype(cryptonote::network_type nettype) { m_network_type = nettype; } + + CHAIN_HTTP_TO_MAP2(cryptonote::core_rpc_server::connection_context); //forward http requests to uri map + BEGIN_URI_MAP2() + MAP_URI_AUTO_JON2("/send_raw_transaction", on_send_raw_tx_2, cryptonote::COMMAND_RPC_SEND_RAW_TX) + MAP_URI_AUTO_JON2("/sendrawtransaction", on_send_raw_tx_2, cryptonote::COMMAND_RPC_SEND_RAW_TX) + else { // Default to parent for non-overriden callbacks + return cryptonote::core_rpc_server::handle_http_request_map(query_info, response_info, m_conn_context); + } + END_URI_MAP2() + + bool on_send_raw_tx_2(const cryptonote::COMMAND_RPC_SEND_RAW_TX::request& req, cryptonote::COMMAND_RPC_SEND_RAW_TX::response& res, const cryptonote::core_rpc_server::connection_context *ctx); + +protected: + cryptonote::network_type m_network_type; +}; + +class mock_daemon { +public: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + typedef nodetool::node_server<t_protocol_raw> t_node_server; + + static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(60); + + cryptonote::core * m_core; + t_protocol_raw m_protocol; + mock_rpc_daemon m_rpc_server; + t_node_server m_server; + cryptonote::network_type m_network_type; + epee::net_utils::http::http_simple_client m_http_client; + + bool m_start_p2p; + bool m_start_zmq; + boost::program_options::variables_map m_vm; + + std::string m_p2p_bind_port; + std::string m_rpc_bind_port; + std::string m_zmq_bind_port; + + std::atomic<bool> m_stopped; + std::atomic<bool> m_terminated; + std::atomic<bool> m_deinitalized; + boost::thread m_run_thread; + + mock_daemon( + cryptonote::core * core, + boost::program_options::variables_map const & vm + ) + : m_core(core) + , m_vm(vm) + , m_start_p2p(false) + , m_start_zmq(false) + , m_terminated(false) + , m_deinitalized(false) + , m_stopped(false) + , m_protocol{*core, nullptr, command_line::get_arg(vm, cryptonote::arg_offline)} + , m_server{m_protocol} + , m_rpc_server{*core, m_server} + { + // Handle circular dependencies + m_protocol.set_p2p_endpoint(&m_server); + m_core->set_cryptonote_protocol(&m_protocol); + load_params(vm); + } + + virtual ~mock_daemon(); + + static void init_options(boost::program_options::options_description & option_spec); + static void default_options(boost::program_options::variables_map & vm); + static void set_ports(boost::program_options::variables_map & vm, unsigned initial_port); + + mock_daemon * set_start_p2p(bool fl) { m_start_p2p = fl; return this; } + mock_daemon * set_start_zmq(bool fl) { m_start_zmq = fl; return this; } + + void init(); + void deinit(); + void run(); + bool run_main(); + void stop(); + void stop_p2p(); + void stop_rpc(); + void init_and_run(); + void stop_and_deinit(); + void try_init_and_run(boost::optional<unsigned> initial_port=boost::none); + + void mine_blocks(size_t num_blocks, const std::string &miner_address); + void start_mining(const std::string &miner_address, uint64_t threads_count=1, bool do_background_mining=false, bool ignore_battery=true); + void stop_mining(); + uint64_t get_height(); + + void load_params(boost::program_options::variables_map const & vm); + + std::string zmq_addr() const { return std::string("127.0.0.1:") + m_zmq_bind_port; } + std::string rpc_addr() const { return std::string("127.0.0.1:") + m_rpc_bind_port; } + std::string p2p_addr() const { return std::string("127.0.0.1:") + m_p2p_bind_port; } + cryptonote::network_type nettype() const { return m_network_type; } + cryptonote::core * core() const { return m_core; } +}; diff --git a/tests/trezor/tools.cpp b/tests/trezor/tools.cpp new file mode 100644 index 000000000..432350cf6 --- /dev/null +++ b/tests/trezor/tools.cpp @@ -0,0 +1,56 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "tools.h" + +namespace tools { + +namespace po = boost::program_options; + +void options::set_option(boost::program_options::variables_map &vm, const std::string & key, const po::variable_value &pv) +{ + auto it = vm.find(key); + if (it == vm.end()) + { + vm.insert(std::make_pair(key, pv)); + } + else + { + it->second = pv; + } +} + +void options::build_options(boost::program_options::variables_map & vm, const po::options_description & desc_params) +{ + const char *argv[2] = {nullptr}; + po::store(po::parse_command_line(1, argv, desc_params), vm); + po::notify(vm); +} + +} + diff --git a/tests/trezor/tools.h b/tests/trezor/tools.h new file mode 100644 index 000000000..d348a5137 --- /dev/null +++ b/tests/trezor/tools.h @@ -0,0 +1,61 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "misc_log_ex.h" +#include "daemon/daemon.h" +#include "rpc/daemon_handler.h" +#include "rpc/zmq_server.h" +#include "common/password.h" +#include "common/util.h" +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "daemon/protocol.h" +#include "daemon/rpc.h" +#include "daemon/command_server.h" +#include "daemon/command_server.h" +#include "daemon/command_line_args.h" +#include "version.h" + +namespace tools { + +class options { +public: + static void set_option(boost::program_options::variables_map &vm, const std::string &key, const boost::program_options::variable_value &pv); + static void build_options(boost::program_options::variables_map & vm, const boost::program_options::options_description & desc_params); + + template<typename T, bool required, bool dependent, int NUM_DEPS> + static void set_option(boost::program_options::variables_map &vm, const command_line::arg_descriptor<T, required, dependent, NUM_DEPS> &arg, const boost::program_options::variable_value &pv) + { + set_option(vm, arg.name, pv); + } +}; + +}; + diff --git a/tests/trezor/trezor_tests.cpp b/tests/trezor/trezor_tests.cpp index c2b46f698..310fa45f1 100644 --- a/tests/trezor/trezor_tests.cpp +++ b/tests/trezor/trezor_tests.cpp @@ -41,6 +41,7 @@ using namespace cryptonote; #include "common/util.h" #include "common/command_line.h" #include "trezor_tests.h" +#include "tools.h" #include "device/device_cold.hpp" #include "device_trezor/device_trezor.hpp" @@ -57,7 +58,7 @@ namespace const command_line::arg_descriptor<bool> arg_fix_chain = {"fix_chain", "If chain_patch is given and file cannot be used, it is ignored and overwriten", false}; } - +#define HW_TREZOR_NAME "Trezor" #define TREZOR_ACCOUNT_ORDERING &m_miner_account, &m_alice_account, &m_bob_account, &m_eve_account #define TREZOR_COMMON_TEST_CASE(genclass, CORE, BASE) \ rollback_chain(CORE, BASE.head_block()); \ @@ -70,14 +71,17 @@ namespace #define TREZOR_SETUP_CHAIN(NAME) do { \ ++tests_count; \ try { \ - setup_chain(&core, trezor_base, chain_path, fix_chain); \ + setup_chain(core, trezor_base, chain_path, fix_chain, vm_core); \ } catch (const std::exception& ex) { \ failed_tests.emplace_back("gen_trezor_base " #NAME); \ } \ } while(0) + +static device_trezor_test *trezor_device = nullptr; +static device_trezor_test *ensure_trezor_test_device(); static void rollback_chain(cryptonote::core * core, const cryptonote::block & head); -static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain); +static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core); int main(int argc, char* argv[]) { @@ -123,38 +127,76 @@ int main(int argc, char* argv[]) const bool heavy_tests = command_line::get_arg(vm, arg_heavy_tests); const bool fix_chain = command_line::get_arg(vm, arg_fix_chain); - hw::trezor::register_all(); + hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device()); + // hw::trezor::register_all(); // We use our shim instead. // Bootstrapping common chain & accounts - cryptonote::core * core = nullptr; + const uint8_t initial_hf = 9; + const uint8_t max_hf = 10; + + cryptonote::core core_obj(nullptr); + cryptonote::core * const core = &core_obj; + std::shared_ptr<mock_daemon> daemon = nullptr; gen_trezor_base trezor_base; trezor_base.setup_args(trezor_path, heavy_tests); - trezor_base.rct_config({rct::RangeProofPaddedBulletproof, 1}); // HF9 tests + trezor_base.set_hard_fork(initial_hf); - TREZOR_SETUP_CHAIN("HF9"); - - // Individual test cases using shared pre-generated blockchain. - TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync, core, trezor_base); + // Arguments for core & daemon + po::variables_map vm_core; + po::options_description desc_params_core("Core"); + mock_daemon::init_options(desc_params_core); + tools::options::build_options(vm_core, desc_params_core); + mock_daemon::default_options(vm_core); // Transaction tests - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_2sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_1norm_2sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base); + for(uint8_t hf=initial_hf; hf <= max_hf; ++hf) + { + MDEBUG("Transaction tests for HF " << (int)hf); + if (hf > initial_hf) + { + daemon->stop_and_deinit(); + daemon = nullptr; + trezor_base.daemon(nullptr); + } + + trezor_base.set_hard_fork(hf); + TREZOR_SETUP_CHAIN(std::string("HF") + std::to_string((int)hf)); + + daemon = std::make_shared<mock_daemon>(core, vm_core); + CHECK_AND_ASSERT_THROW_MES(daemon->nettype() == trezor_base.nettype(), "Serialized chain network type does not match"); + + daemon->try_init_and_run(); + trezor_base.daemon(daemon); + + // Hard-fork independent tests + if (hf == initial_hf) + { + TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_without_refresh, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_live_refresh, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_with_refresh, core, trezor_base); + } + + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_2sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_1norm_2sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base); + TREZOR_COMMON_TEST_CASE(wallet_api_tests, core, trezor_base); + } if (trezor_base.heavy_tests()) { TREZOR_COMMON_TEST_CASE(gen_trezor_many_utxo, core, trezor_base); } + daemon->stop(); core->deinit(); el::Level level = (failed_tests.empty() ? el::Level::Info : el::Level::Error); MLOG(level, "\nREPORT:"); @@ -243,7 +285,36 @@ static bool serialize_chain_to_file(std::vector<test_event_entry>& events, gen_t CATCH_ENTRY_L0("serialize_chain_to_file", false); } -static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain) +template<class t_test_class> +static bool init_core_replay_events(std::vector<test_event_entry>& events, cryptonote::core * core, const po::variables_map & vm_core) +{ + // this test needs for it to be so. + get_test_options<t_test_class> gto; + + // Hardforks can be specified in events. + v_hardforks_t hardforks; + cryptonote::test_options test_options_tmp{}; + const cryptonote::test_options * test_options_ = >o.test_options; + if (extract_hard_forks(events, hardforks)){ + hardforks.push_back(std::make_pair((uint8_t)0, (uint64_t)0)); // terminator + test_options_tmp.hard_forks = hardforks.data(); + test_options_ = &test_options_tmp; + } + + core->deinit(); + CHECK_AND_ASSERT_THROW_MES(core->init(vm_core, test_options_), "Core init failed"); + core->get_blockchain_storage().get_db().set_batch_transactions(true); + + // start with a clean pool + std::vector<crypto::hash> pool_txs; + CHECK_AND_ASSERT_THROW_MES(core->get_pool_transaction_hashes(pool_txs), "Failed to flush txpool"); + core->get_blockchain_storage().flush_txes_from_pool(pool_txs); + + t_test_class validator; + return replay_events_through_core<t_test_class>(*core, events, validator); +} + +static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core) { std::vector<test_event_entry> events; const bool do_serialize = !chain_path.empty(); @@ -272,6 +343,7 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, { try { + trezor_base.clear(); generated = trezor_base.generate(events); if (generated && !loaded && do_serialize) @@ -287,7 +359,7 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, } trezor_base.fix_hf(events); - if (generated && do_replay_events_get_core<gen_trezor_base>(events, core)) + if (generated && init_core_replay_events<gen_trezor_base>(events, core, vm_core)) { MGINFO_GREEN("#TEST-chain-init# Succeeded "); } @@ -298,6 +370,14 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, } } +static device_trezor_test *ensure_trezor_test_device(){ + if (!trezor_device) { + trezor_device = new device_trezor_test(); + trezor_device->set_name(HW_TREZOR_NAME); + } + return trezor_device; +} + static void add_hforks(std::vector<test_event_entry>& events, const v_hardforks_t& hard_forks) { event_replay_settings repl_set; @@ -483,8 +563,20 @@ static std::vector<tools::wallet2*> vct_wallets(tools::wallet2* w1=nullptr, tool return res; } +static uint64_t get_available_funds(tools::wallet2* wallet, uint32_t account=0) +{ + tools::wallet2::transfer_container transfers; + wallet->get_transfers(transfers); + uint64_t sum = 0; + for(const auto & cur : transfers) + { + sum += !cur.m_spent && cur.m_subaddr_index.major == account ? cur.amount() : 0; + } + return sum; +} + // gen_trezor_base -const uint64_t gen_trezor_base::m_ts_start = 1338224400; +const uint64_t gen_trezor_base::m_ts_start = 1397862000; // As default wallet timestamp is 1397516400 const uint64_t gen_trezor_base::m_wallet_ts = m_ts_start - 60*60*24*4; const std::string gen_trezor_base::m_device_name = "Trezor:udp"; const std::string gen_trezor_base::m_master_seed_str = "14821d0bc5659b24cafbc889dc4fc60785ee08b65d71c525f81eeaba4f3a570f"; @@ -494,12 +586,16 @@ const std::string gen_trezor_base::m_alice_view_private = "a6ccd4ac344a295d1387f gen_trezor_base::gen_trezor_base(){ m_rct_config = {rct::RangeProofPaddedBulletproof, 1}; + m_test_get_tx_key = true; + m_network_type = cryptonote::TESTNET; } gen_trezor_base::gen_trezor_base(const gen_trezor_base &other): m_generator(other.m_generator), m_bt(other.m_bt), m_miner_account(other.m_miner_account), m_bob_account(other.m_bob_account), m_alice_account(other.m_alice_account), m_eve_account(other.m_eve_account), - m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config) + m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config), + m_heavy_tests(other.m_heavy_tests), m_test_get_tx_key(other.m_test_get_tx_key), m_live_refresh_enabled(other.m_live_refresh_enabled), + m_network_type(other.m_network_type), m_daemon(other.m_daemon) { } @@ -513,33 +609,27 @@ void gen_trezor_base::setup_args(const std::string & trezor_path, bool heavy_tes void gen_trezor_base::setup_trezor() { hw::device &hwdev = hw::get_device(m_trezor_path); - m_trezor = dynamic_cast<hw::trezor::device_trezor *>(&hwdev); - CHECK_AND_ASSERT_THROW_MES(m_trezor, "Dynamic cast failed"); - - m_trezor->set_debug(true); // debugging commands on Trezor (auto-confirm transactions) - - CHECK_AND_ASSERT_THROW_MES(m_trezor->set_name(m_trezor_path), "Could not set device name " << m_trezor_path); - m_trezor->set_network_type(MAINNET); - m_trezor->set_derivation_path(""); // empty derivation path + auto trezor = dynamic_cast<device_trezor_test *>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(trezor, "Dynamic cast failed"); - CHECK_AND_ASSERT_THROW_MES(m_trezor->init(), "Could not initialize the device " << m_trezor_path); - CHECK_AND_ASSERT_THROW_MES(m_trezor->connect(), "Could not connect to the device " << m_trezor_path); - m_trezor->wipe_device(); - m_trezor->load_device(m_device_seed); - m_trezor->release(); - m_trezor->disconnect(); + trezor->setup_for_tests(m_trezor_path, m_device_seed, m_network_type); + m_trezor = trezor; } void gen_trezor_base::fork(gen_trezor_base & other) { other.m_generator = m_generator; other.m_bt = m_bt; + other.m_network_type = m_network_type; + other.m_daemon = m_daemon; other.m_events = m_events; other.m_head = m_head; other.m_hard_forks = m_hard_forks; other.m_trezor_path = m_trezor_path; other.m_heavy_tests = m_heavy_tests; other.m_rct_config = m_rct_config; + other.m_test_get_tx_key = m_test_get_tx_key; + other.m_live_refresh_enabled = m_live_refresh_enabled; other.m_miner_account = m_miner_account; other.m_bob_account = m_bob_account; @@ -577,10 +667,22 @@ void gen_trezor_base::init_fields() m_alice_account.set_createtime(m_wallet_ts); } +void gen_trezor_base::update_client_settings() +{ + auto dev_trezor = dynamic_cast<::hw::trezor::device_trezor*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_trezor, "Could not cast to device_trezor"); + + dev_trezor->set_live_refresh_enabled(m_live_refresh_enabled); +} + bool gen_trezor_base::generate(std::vector<test_event_entry>& events) { init_fields(); setup_trezor(); + + m_live_refresh_enabled = false; + update_client_settings(); + m_alice_account.create_from_device(*m_trezor); m_alice_account.set_createtime(m_wallet_ts); @@ -589,7 +691,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) cryptonote::block blk_gen; std::vector<size_t> block_weights; - generate_genesis_block(blk_gen, get_config(MAINNET).GENESIS_TX, get_config(MAINNET).GENESIS_NONCE); + generate_genesis_block(blk_gen, get_config(m_network_type).GENESIS_TX, get_config(m_network_type).GENESIS_NONCE); events.push_back(blk_gen); generator.add_block(blk_gen, 0, block_weights, 0); @@ -644,8 +746,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) MDEBUG("Hardfork height: " << hardfork_height << " at block: " << get_block_hash(blk_4r)); // RCT transactions, wallets have to be used, wallet init - m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true)); + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); @@ -659,7 +761,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) auto addr_alice_sub_1_2 = m_wl_alice->get_subaddress({1, 2}); // Miner -> Bob, RCT funds - MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(5), 10, blk_4); + MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(50), 10, blk_4); const size_t target_rct = m_heavy_tests ? 105 : 15; for(size_t i = 0; i < target_rct; ++i) @@ -688,9 +790,9 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) // Simple RCT transactions MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(7), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(1), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(3), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(4), 10, blk_4); + MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(10), 10, blk_4); + MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(30), 10, blk_4); + MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(40), 10, blk_4); MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_5, blk_4r, m_miner_account, txs_blk_5, CUR_HF); // Simple transaction check @@ -704,6 +806,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) // RCT transactions, wallets have to be used wallet_tools::process_transactions(m_wl_alice.get(), events, blk_5r, m_bt); wallet_tools::process_transactions(m_wl_bob.get(), events, blk_5r, m_bt); + MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get())); + MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get())); // Send Alice -> Bob, manually constructed. Simple TX test, precondition. cryptonote::transaction tx_1; @@ -768,18 +872,21 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events) m_eve_account.set_createtime(m_wallet_ts); setup_trezor(); + update_client_settings(); m_alice_account.create_from_device(*m_trezor); m_alice_account.set_createtime(m_wallet_ts); - m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_eve.reset(new tools::wallet2(MAINNET, 1, true)); + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); wallet_tools::process_transactions(m_wl_alice.get(), events, m_head, m_bt); wallet_tools::process_transactions(m_wl_bob.get(), events, m_head, m_bt); + MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get())); + MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get())); } void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events) @@ -804,12 +911,14 @@ void gen_trezor_base::test_setup(std::vector<test_event_entry>& events) add_shared_events(events); setup_trezor(); + update_client_settings(); + m_alice_account.create_from_device(*m_trezor); m_alice_account.set_createtime(m_wallet_ts); - m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_eve.reset(new tools::wallet2(MAINNET, 1, true)); + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); @@ -818,13 +927,11 @@ void gen_trezor_base::test_setup(std::vector<test_event_entry>& events) wallet_tools::process_transactions(m_wl_eve.get(), events, m_head, m_bt); } -void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep) +void gen_trezor_base::add_transactions_to_events( + std::vector<test_event_entry>& events, + test_generator &generator, + const std::vector<cryptonote::transaction> &txs) { - // Construct pending transaction for signature in the Trezor. - const uint64_t height_pre = num_blocks(events) - 1; - cryptonote::block head_block = get_head_block(events); - const crypto::hash head_hash = get_block_hash(head_block); - // If current test requires higher hard-fork, move it up const auto current_hf = m_hard_forks.back().first; const uint8_t tx_hf = m_rct_config.bp_version == 2 ? 10 : 9; @@ -832,8 +939,28 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: throw std::runtime_error("Too late for HF change"); } - tools::wallet2::unsigned_tx_set txs; std::list<cryptonote::transaction> tx_list; + for(const auto & tx : txs) + { + events.push_back(tx); + tx_list.push_back(tx); + } + + MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_new, m_head, m_miner_account, tx_list, tx_hf); + MDEBUG("New tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_new)); + + m_head = blk_new; +} + +void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep) +{ + // Construct pending transaction for signature in the Trezor. + const uint64_t height_pre = num_blocks(events) - 1; + cryptonote::block head_block = get_head_block(events); + const crypto::hash head_hash = get_block_hash(head_block); + + tools::wallet2::unsigned_tx_set txs; + std::vector<cryptonote::transaction> tx_list; for(auto &ptx : ptxs) { txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(ptx, *m_trezor)); @@ -848,6 +975,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: hw::wallet_shim wallet_shim; setup_shim(&wallet_shim); aux_data.tx_recipients = dsts_info; + aux_data.bp_version = m_rct_config.bp_version; dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); MDEBUG("Signed tx data from hw: " << exported_txs.ptx.size() << " transactions"); @@ -865,13 +993,11 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: CHECK_AND_ASSERT_THROW_MES(resx, "Trezor tx_1 semantics failed"); CHECK_AND_ASSERT_THROW_MES(resy, "Trezor tx_1 Nonsemantics failed"); - events.push_back(c_ptx.tx); tx_list.push_back(c_ptx.tx); MDEBUG("Transaction: " << dump_data(c_ptx.tx)); } - MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_7, m_head, m_miner_account, tx_list, tx_hf); - MDEBUG("Trezor tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_7)); + add_transactions_to_events(events, generator, tx_list); // TX receive test uint64_t sum_in = 0; @@ -909,7 +1035,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: const bool sender = widx == 0; tools::wallet2 *wl = wallets[widx]; - wallet_tools::process_transactions(wl, events, blk_7, m_bt, boost::make_optional(head_hash)); + wallet_tools::process_transactions(wl, events, m_head, m_bt, boost::make_optional(head_hash)); tools::wallet2::transfer_container m_trans; tools::wallet2::transfer_container m_trans_txid; @@ -972,6 +1098,120 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: } CHECK_AND_ASSERT_THROW_MES(sum_in == sum_out, "Tx amount mismatch"); + + // Test get_tx_key feature for stored private tx keys + test_get_tx(events, wallets, exported_txs.ptx, aux_data.tx_device_aux); +} + +bool gen_trezor_base::verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs) +{ + ::crypto::public_key tx_pub_c; + ::crypto::secret_key_to_public_key(tx_priv, tx_pub_c); + if (tx_pub == tx_pub_c) + return true; + + for(const auto & elem : subs) + { + tx_pub_c = rct::rct2pk(rct::scalarmultKey(rct::pk2rct(elem.first), rct::sk2rct(tx_priv))); + if (tx_pub == tx_pub_c) + return true; + } + return false; +} + +void gen_trezor_base::test_get_tx( + std::vector<test_event_entry>& events, + std::vector<tools::wallet2*> wallets, + const std::vector<tools::wallet2::pending_tx> &ptxs, + const std::vector<std::string> &aux_tx_info) +{ + if (!m_test_get_tx_key) + { + return; + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + if (!dev_cold->is_get_tx_key_supported()) + { + MERROR("Get TX key is not supported by the connected Trezor"); + return; + } + + subaddresses_t all_subs; + for(tools::wallet2 * wlt : wallets) + { + wlt->expand_subaddresses({10, 20}); + + const subaddresses_t & cur_sub = wallet_accessor_test::get_subaddresses(wlt); + all_subs.insert(cur_sub.begin(), cur_sub.end()); + } + + for(size_t txid = 0; txid < ptxs.size(); ++txid) + { + const auto &c_ptx = ptxs[txid]; + const auto &c_tx = c_ptx.tx; + const ::crypto::hash tx_prefix_hash = cryptonote::get_transaction_prefix_hash(c_tx); + + auto tx_pub = cryptonote::get_tx_pub_key_from_extra(c_tx.extra); + auto additional_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(c_tx.extra); + + hw::device_cold:: tx_key_data_t tx_key_data; + std::vector<::crypto::secret_key> tx_keys; + + dev_cold->load_tx_key_data(tx_key_data, aux_tx_info[txid]); + CHECK_AND_ASSERT_THROW_MES(std::string(tx_prefix_hash.data, 32) == tx_key_data.tx_prefix_hash, "TX prefix mismatch"); + + dev_cold->get_tx_key(tx_keys, tx_key_data, m_alice_account.get_keys().m_view_secret_key); + CHECK_AND_ASSERT_THROW_MES(!tx_keys.empty(), "Empty TX keys"); + CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[0], tx_pub, all_subs), "Tx pub mismatch"); + CHECK_AND_ASSERT_THROW_MES(additional_pub_keys.size() == tx_keys.size() - 1, "Invalid additional keys count"); + + for(size_t i = 0; i < additional_pub_keys.size(); ++i) + { + CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[i + 1], additional_pub_keys[i], all_subs), "Tx pub mismatch"); + } + } +} + +void gen_trezor_base::mine_and_test(std::vector<test_event_entry>& events) +{ + cryptonote::core * core = daemon()->core(); + const uint64_t height_before_mining = daemon()->get_height(); + + const auto miner_address = cryptonote::get_account_address_as_str(FAKECHAIN, false, get_address(m_miner_account)); + daemon()->mine_blocks(1, miner_address); + + const uint64_t cur_height = daemon()->get_height(); + CHECK_AND_ASSERT_THROW_MES(height_before_mining < cur_height, "Mining fail"); + + const crypto::hash top_hash = core->get_blockchain_storage().get_block_id_by_height(height_before_mining); + cryptonote::block top_block{}; + CHECK_AND_ASSERT_THROW_MES(core->get_blockchain_storage().get_block_by_hash(top_hash, top_block), "Block fetch fail"); + CHECK_AND_ASSERT_THROW_MES(!top_block.tx_hashes.empty(), "Mined block is empty"); + + std::vector<cryptonote::transaction> txs_found; + std::vector<crypto::hash> txs_missed; + bool r = core->get_blockchain_storage().get_transactions(top_block.tx_hashes, txs_found, txs_missed); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction lookup fail"); + CHECK_AND_ASSERT_THROW_MES(!txs_found.empty(), "Transaction lookup fail"); + + // Transaction is not expanded, but mining verified it. + events.push_back(txs_found[0]); + events.push_back(top_block); +} + +void gen_trezor_base::set_hard_fork(uint8_t hf) +{ + m_top_hard_fork = hf; + if (hf < 9){ + throw std::runtime_error("Minimal supported Hardfork is 9"); + } else if (hf == 9){ + rct_config({rct::RangeProofPaddedBulletproof, 1}); + } else { + rct_config({rct::RangeProofPaddedBulletproof, 2}); + } } #define TREZOR_TEST_PREFIX() \ @@ -1182,6 +1422,48 @@ std::vector<tools::wallet2::pending_tx> tsx_builder::build() return m_ptxs; } +device_trezor_test::device_trezor_test(): m_tx_sign_ctr(0), m_compute_key_image_ctr(0) {} + +void device_trezor_test::clear_test_counters(){ + m_tx_sign_ctr = 0; + m_compute_key_image_ctr = 0; +} + +void device_trezor_test::setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type){ + this->clear_test_counters(); + this->set_callback(nullptr); + this->set_debug(true); // debugging commands on Trezor (auto-confirm transactions) + + CHECK_AND_ASSERT_THROW_MES(this->set_name(trezor_path), "Could not set device name " << trezor_path); + this->set_network_type(network_type); + this->set_derivation_path(""); // empty derivation path + + CHECK_AND_ASSERT_THROW_MES(this->init(), "Could not initialize the device " << trezor_path); + CHECK_AND_ASSERT_THROW_MES(this->connect(), "Could not connect to the device " << trezor_path); + this->wipe_device(); + this->load_device(seed); + this->release(); + this->disconnect(); +} + +bool device_trezor_test::compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key, + const ::crypto::key_derivation &recv_derivation, size_t real_output_index, + const ::cryptonote::subaddress_index &received_index, + ::cryptonote::keypair &in_ephemeral, ::crypto::key_image &ki) { + + bool res = device_trezor::compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, + in_ephemeral, ki); + m_compute_key_image_ctr += res; + return res; +} + +void +device_trezor_test::tx_sign(hw::wallet_shim *wallet, const ::tools::wallet2::unsigned_tx_set &unsigned_tx, size_t idx, + hw::tx_aux_data &aux_data, std::shared_ptr<hw::trezor::protocol::tx::Signer> &signer) { + m_tx_sign_ctr += 1; + device_trezor::tx_sign(wallet, unsigned_tx, idx, aux_data, signer); +} + bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events) { test_generator generator(m_generator); @@ -1207,6 +1489,92 @@ bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events) uint64_t spent = 0, unspent = 0; m_wl_alice->import_key_images(ski, 0, spent, unspent, false); + + auto dev_trezor_test = dynamic_cast<device_trezor_test*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement test interface"); + if (!m_live_refresh_enabled) + CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == 0, "Live refresh should not happen: " << dev_trezor_test->m_compute_key_image_ctr); + else + CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == ski.size(), "Live refresh counts invalid"); + + return true; +} + +bool gen_trezor_ki_sync_with_refresh::generate(std::vector<test_event_entry>& events) +{ + m_live_refresh_enabled = true; + return gen_trezor_ki_sync::generate(events); +} + +bool gen_trezor_ki_sync_without_refresh::generate(std::vector<test_event_entry>& events) +{ + m_live_refresh_enabled = false; + return gen_trezor_ki_sync::generate(events); +} + +bool gen_trezor_live_refresh::generate(std::vector<test_event_entry>& events) +{ + test_generator generator(m_generator); + test_setup(events); + + auto dev_cold = dynamic_cast<::hw::device_cold*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + if (!dev_cold->is_live_refresh_supported()){ + MDEBUG("Trezor does not support live refresh"); + return true; + } + + hw::device & sw_device = hw::get_device("default"); + + dev_cold->live_refresh_start(); + for(unsigned i=0; i<50; ++i) + { + cryptonote::subaddress_index subaddr = {0, i}; + + ::crypto::secret_key r; + ::crypto::public_key R; + ::crypto::key_derivation D; + ::crypto::public_key pub_ver; + ::crypto::key_image ki; + + ::crypto::random32_unbiased((unsigned char*)r.data); + ::crypto::secret_key_to_public_key(r, R); + memcpy(D.data, rct::scalarmultKey(rct::pk2rct(R), rct::sk2rct(m_alice_account.get_keys().m_view_secret_key)).bytes, 32); + + ::crypto::secret_key scalar_step1; + ::crypto::secret_key scalar_step2; + ::crypto::derive_secret_key(D, i, m_alice_account.get_keys().m_spend_secret_key, scalar_step1); + if (i == 0) + { + scalar_step2 = scalar_step1; + } + else + { + ::crypto::secret_key subaddr_sk = sw_device.get_subaddress_secret_key(m_alice_account.get_keys().m_view_secret_key, subaddr); + sw_device.sc_secret_add(scalar_step2, scalar_step1, subaddr_sk); + } + + ::crypto::secret_key_to_public_key(scalar_step2, pub_ver); + ::crypto::generate_key_image(pub_ver, scalar_step2, ki); + + cryptonote::keypair in_ephemeral; + ::crypto::key_image ki2; + + dev_cold->live_refresh( + m_alice_account.get_keys().m_view_secret_key, + pub_ver, + D, + i, + subaddr, + in_ephemeral, + ki2 + ); + + CHECK_AND_ASSERT_THROW_MES(ki == ki2, "Key image inconsistent"); + } + + dev_cold->live_refresh_finish(); return true; } @@ -1407,3 +1775,62 @@ bool gen_trezor_many_utxo::generate(std::vector<test_event_entry>& events) TREZOR_TEST_SUFFIX(); } +void wallet_api_tests::init() +{ + m_wallet_dir = boost::filesystem::unique_path(); + boost::filesystem::create_directories(m_wallet_dir); +} + +wallet_api_tests::~wallet_api_tests() +{ + try + { + if (!m_wallet_dir.empty() && boost::filesystem::exists(m_wallet_dir)) + { + boost::filesystem::remove_all(m_wallet_dir); + } + } + catch(...) + { + MERROR("Could not remove wallet directory"); + } +} + +bool wallet_api_tests::generate(std::vector<test_event_entry>& events) +{ + init(); + test_setup(events); + const std::string wallet_path = (m_wallet_dir / "wallet").string(); + const auto api_net_type = m_network_type == TESTNET ? Monero::TESTNET : Monero::MAINNET; + + Monero::WalletManager *wmgr = Monero::WalletManagerFactory::getWalletManager(); + std::unique_ptr<Monero::Wallet> w{wmgr->createWalletFromDevice(wallet_path, "", api_net_type, m_trezor_path, 1)}; + CHECK_AND_ASSERT_THROW_MES(w->init(daemon()->rpc_addr(), 0), "Wallet init fail"); + CHECK_AND_ASSERT_THROW_MES(w->refresh(), "Refresh fail"); + uint64_t balance = w->balance(0); + MINFO("Balance: " << balance); + CHECK_AND_ASSERT_THROW_MES(w->status() == Monero::PendingTransaction::Status_Ok, "Status nok"); + + auto addr = get_address(m_eve_account); + auto recepient_address = cryptonote::get_account_address_as_str(m_network_type, false, addr); + Monero::PendingTransaction * transaction = w->createTransaction(recepient_address, + "", + MK_COINS(10), + TREZOR_TEST_MIXIN, + Monero::PendingTransaction::Priority_Medium, + 0, + std::set<uint32_t>{}); + CHECK_AND_ASSERT_THROW_MES(transaction->status() == Monero::PendingTransaction::Status_Ok, "Status nok"); + w->refresh(); + + CHECK_AND_ASSERT_THROW_MES(w->balance(0) == balance, "Err"); + CHECK_AND_ASSERT_THROW_MES(transaction->amount() == MK_COINS(10), "Err"); + CHECK_AND_ASSERT_THROW_MES(transaction->commit(), "Err"); + CHECK_AND_ASSERT_THROW_MES(w->balance(0) != balance, "Err"); + CHECK_AND_ASSERT_THROW_MES(wmgr->closeWallet(w.get()), "Err"); + (void)w.release(); + + mine_and_test(events); + return true; +} + diff --git a/tests/trezor/trezor_tests.h b/tests/trezor/trezor_tests.h index 41db1cce5..bed49fec4 100644 --- a/tests/trezor/trezor_tests.h +++ b/tests/trezor/trezor_tests.h @@ -31,6 +31,8 @@ #pragma once #include <device_trezor/device_trezor.hpp> +#include <wallet/api/wallet2_api.h> +#include "daemon.h" #include "../core_tests/chaingen.h" #include "../core_tests/wallet_tools.h" @@ -50,29 +52,48 @@ public: gen_trezor_base(const gen_trezor_base &other); virtual ~gen_trezor_base() {}; - void setup_args(const std::string & trezor_path, bool heavy_tests=false); + virtual void setup_args(const std::string & trezor_path, bool heavy_tests=false); virtual bool generate(std::vector<test_event_entry>& events); virtual void load(std::vector<test_event_entry>& events); // load events, init test obj - void fix_hf(std::vector<test_event_entry>& events); - void update_trackers(std::vector<test_event_entry>& events); + virtual void fix_hf(std::vector<test_event_entry>& events); + virtual void update_trackers(std::vector<test_event_entry>& events); - void fork(gen_trezor_base & other); // fork generated chain to another test - void clear(); // clears m_events, bt, generator, hforks - void add_shared_events(std::vector<test_event_entry>& events); // m_events -> events - void test_setup(std::vector<test_event_entry>& events); // init setup env, wallets + virtual void fork(gen_trezor_base & other); // fork generated chain to another test + virtual void clear(); // clears m_events, bt, generator, hforks + virtual void add_shared_events(std::vector<test_event_entry>& events); // m_events -> events + virtual void test_setup(std::vector<test_event_entry>& events); // init setup env, wallets - void test_trezor_tx(std::vector<test_event_entry>& events, + virtual void add_transactions_to_events( + std::vector<test_event_entry>& events, + test_generator &generator, + const std::vector<cryptonote::transaction> &txs); + + virtual void test_trezor_tx( + std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep=false); - crypto::hash head_hash() { return get_block_hash(m_head); } - cryptonote::block head_block() { return m_head; } - bool heavy_tests() { return m_heavy_tests; } + virtual void test_get_tx( + std::vector<test_event_entry>& events, + std::vector<tools::wallet2*> wallets, + const std::vector<tools::wallet2::pending_tx> &ptxs, + const std::vector<std::string> &aux_tx_info); + + virtual void mine_and_test(std::vector<test_event_entry>& events); + + virtual void set_hard_fork(uint8_t hf); + + crypto::hash head_hash() const { return get_block_hash(m_head); } + cryptonote::block head_block() const { return m_head; } + bool heavy_tests() const { return m_heavy_tests; } void rct_config(rct::RCTConfig rct_config) { m_rct_config = rct_config; } - uint8_t cur_hf(){ return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; } + uint8_t cur_hf() const { return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; } + cryptonote::network_type nettype() const { return m_network_type; } + std::shared_ptr<mock_daemon> daemon() const { return m_daemon; } + void daemon(std::shared_ptr<mock_daemon> daemon){ m_daemon = std::move(daemon); } // Static configuration static const uint64_t m_ts_start; @@ -84,19 +105,26 @@ public: static const std::string m_alice_view_private; protected: - void setup_trezor(); - void init_fields(); + virtual void setup_trezor(); + virtual void init_fields(); + virtual void update_client_settings(); + virtual bool verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs); test_generator m_generator; block_tracker m_bt; + cryptonote::network_type m_network_type; + std::shared_ptr<mock_daemon> m_daemon; + uint8_t m_top_hard_fork; v_hardforks_t m_hard_forks; cryptonote::block m_head; std::vector<test_event_entry> m_events; std::string m_trezor_path; bool m_heavy_tests; + bool m_test_get_tx_key; rct::RCTConfig m_rct_config; + bool m_live_refresh_enabled; cryptonote::account_base m_miner_account; cryptonote::account_base m_bob_account; @@ -113,6 +141,7 @@ protected: void serialize(Archive & ar, const unsigned int /*version*/) { ar & m_generator; + ar & m_network_type; } }; @@ -169,12 +198,52 @@ protected: rct::RCTConfig m_rct_config; }; +// Trezor device ship to track actual method calls. +class device_trezor_test : public hw::trezor::device_trezor { +public: + size_t m_tx_sign_ctr; + size_t m_compute_key_image_ctr; + + device_trezor_test(); + + void clear_test_counters(); + void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type); + + bool compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key, + const ::crypto::key_derivation &recv_derivation, size_t real_output_index, + const ::cryptonote::subaddress_index &received_index, ::cryptonote::keypair &in_ephemeral, + ::crypto::key_image &ki) override; + +protected: + void tx_sign(hw::wallet_shim *wallet, const ::tools::wallet2::unsigned_tx_set &unsigned_tx, size_t idx, + hw::tx_aux_data &aux_data, std::shared_ptr<hw::trezor::protocol::tx::Signer> &signer) override; +}; + +// Tests class gen_trezor_ki_sync : public gen_trezor_base { public: bool generate(std::vector<test_event_entry>& events) override; }; +class gen_trezor_ki_sync_with_refresh : public gen_trezor_ki_sync +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + +class gen_trezor_ki_sync_without_refresh : public gen_trezor_ki_sync +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + +class gen_trezor_live_refresh : public gen_trezor_base +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + class gen_trezor_1utxo : public gen_trezor_base { public: @@ -246,3 +315,15 @@ class gen_trezor_many_utxo : public gen_trezor_base public: bool generate(std::vector<test_event_entry>& events) override; }; + +// Wallet::API tests +class wallet_api_tests : public gen_trezor_base +{ +public: + virtual ~wallet_api_tests(); + void init(); + bool generate(std::vector<test_event_entry>& events) override; + +protected: + boost::filesystem::path m_wallet_dir; +};
\ No newline at end of file diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index b355d566d..56a1f8c4d 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -43,6 +43,7 @@ set(unit_tests_sources crypto.cpp decompose_amount_into_digits.cpp device.cpp + difficulty.cpp dns_resolver.cpp epee_boosted_tcp_server.cpp epee_levin_protocol_handler_async.cpp @@ -52,10 +53,12 @@ set(unit_tests_sources json_serialization.cpp get_xtype_from_string.cpp hashchain.cpp + hmac_keccak.cpp http.cpp keccak.cpp logging.cpp long_term_block_weight.cpp + lmdb.cpp main.cpp memwipe.cpp mlocker.cpp @@ -101,6 +104,7 @@ target_link_libraries(unit_tests cryptonote_protocol cryptonote_core blockchain_db + lmdb_lib rpc net serialization diff --git a/tests/unit_tests/difficulty.cpp b/tests/unit_tests/difficulty.cpp new file mode 100644 index 000000000..090fecc84 --- /dev/null +++ b/tests/unit_tests/difficulty.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "gtest/gtest.h" +#include "cryptonote_basic/difficulty.h" + +static cryptonote::difficulty_type MKDIFF(uint64_t high, uint64_t low) +{ + cryptonote::difficulty_type d = high; + d = (d << 64) | low; + return d; +} + +static crypto::hash MKHASH(uint64_t high, uint64_t low) +{ + cryptonote::difficulty_type hash_target = high; + hash_target = (hash_target << 64) | low; + boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; + crypto::hash h; + ((uint64_t*)&h)[0] = hash_value.convert_to<uint64_t>(); + hash_value >>= 64; + ((uint64_t*)&h)[1] = hash_value.convert_to<uint64_t>(); + hash_value >>= 64; + ((uint64_t*)&h)[2] = hash_value.convert_to<uint64_t>(); + hash_value >>= 64; + ((uint64_t*)&h)[3] = hash_value.convert_to<uint64_t>(); + return h; +} + +TEST(difficulty, check_hash) +{ + ASSERT_TRUE(cryptonote::check_hash(MKHASH(0, 1), MKDIFF(0, 1))); + ASSERT_FALSE(cryptonote::check_hash(MKHASH(0, 1), MKDIFF(0, 2))); + + ASSERT_TRUE(cryptonote::check_hash(MKHASH(0, 0xffffffffffffffff), MKDIFF(0, 0xffffffffffffffff))); + ASSERT_FALSE(cryptonote::check_hash(MKHASH(0, 0xffffffffffffffff), MKDIFF(1, 0))); + + ASSERT_TRUE(cryptonote::check_hash(MKHASH(1, 1), MKDIFF(1, 1))); + ASSERT_FALSE(cryptonote::check_hash(MKHASH(1, 1), MKDIFF(1, 2))); + + ASSERT_TRUE(cryptonote::check_hash(MKHASH(0xffffffffffffffff, 1), MKDIFF(0xffffffffffffffff, 1))); + ASSERT_FALSE(cryptonote::check_hash(MKHASH(0xffffffffffffffff, 1), MKDIFF(0xffffffffffffffff, 2))); +} diff --git a/tests/unit_tests/hmac_keccak.cpp b/tests/unit_tests/hmac_keccak.cpp new file mode 100644 index 000000000..cb35d272a --- /dev/null +++ b/tests/unit_tests/hmac_keccak.cpp @@ -0,0 +1,152 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "gtest/gtest.h" +#include "../io.h" + +extern "C" { +#include "crypto/hmac-keccak.h" +} + +#define KECCAK_BLOCKLEN 136 + +static const struct { + const char *key; + const char *inp; + const char *res; +} keccak_hmac_vectors[] = { + {"", + "", + "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b", + }, + {"00", + "", + "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b", + }, + {"00000000000000000000000000000000", + "", + "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b", + }, + {"", + "00", + "402541d5d678ad30217023a12efbd2f367388dc0253ccddb8f915d187e03d414", + }, + {"0000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "d164c38ca454176281b3c09dc83cf70bd00290835ff5490356993d368cde0577", + }, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0000000000000000000000000000000000000000000000000000000000000000", + "d52c9100301a9546d8c97d7f32e8178fd5f4b0a7646ab761db9e0f503f8b0542", + }, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "d52c9100301a9546d8c97d7f32e8178fd5f4b0a7646ab761db9e0f503f8b0542", + "5b9d98c5d3249176230d70fdb215ee2f93e4783064c9564384ddc396e63c18cb", + }, + {"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "98d033ba0c95bb3aaf5709733443cbedba20f6451b710ccba4118fc5523013c8", + }, + {"65768798a9bacbdcedfe0f2031425364758697a8b9cadbecfd0e1f30415263748596a7b8c9daebfc0d1e2f405162738495a6b7c8d9eafb0c1d2e3f5061728394a5b6c7d8e9fa0b1c2d3e4f60718293a4b5c6d7e8f90a1b2c3d4e5f708192a3b4c5d6e7f8091a2b3c4d5e6f8091a2b3c4d5e6f708192a3b4c5d6e7f90a1b2c3d4e5f60718293a4b5c6d7e8fa0b1c2d3e4f5061728394a5b6c7d8e9fb0c1d2e3f405162738495a6b7c8d9eafc0d1e2f30415263748596a7b8c9daebfd0e1f2031425364758697a8b9cadbecfe0f102132435465768798a9bacbdcedff00112233445566778899aabbccddeef00112233445566778899aabbccddeeff102132435465768798a9bacbdcedfe0f2031425364758697a8b9cadbecfd0e1f30415263748596a7b8c9daebfc0d1e2f405162738495a6b7c8d9eafb0c1d2e3f5061728394", + "7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b2835424f5c697683909daab7c4d1deebf805121f2c394653606d7a8794a1aebbc8d5e2effc091623303d4a5764717e8b98a5b2bfccd9e6f3000d1a2734414e5b6875828f9ca9b6c3d0ddeaf704111e2b3845525f6c798693a0adbac7d4e1eefb0815222f3c495663707d8a97a4b1becbd8e5f2ff0c192633404d5a6774818e9ba8b5c2cfdce9f603101d2a3744515e6b7885929facb9c6d3e0edfa0714212e3b4855626f7c8996a3b0bdcad7e4f1fe0b1825323f4c596673808d9aa7b4c1cedbe8f5020f1c293643505d6a7784919eabb8c5d2dfecf90613202d3a4754616e7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b2835424f5c697683909daab7c4d1deebf805121f2c394653606d7a8794a1ae", + "117b60a7f474fd2245adff82a69429367de8f5263fa96c411986009a743b0e70", + }, + {"00112233445566778899aabbccddeeff102132435465768798a9bacbdcedfe0f", + "000d1a2734414e5b6875828f9ca9b6c3d0ddeaf704111e2b3845525f6c798693a0adbac7d4e1eefb0815222f3c495663707d8a97a4b1becbd8e5f2ff0c192633404d5a6774818e9ba8b5c2cfdce9f603101d2a3744515e6b7885929facb9c6d3e0edfa0714212e3b4855626f7c8996a3b0bdcad7e4f1fe0b1825323f4c596673808d9aa7b4c1cedbe8f5020f1c293643505d6a7784919eabb8c5d2dfecf90613202d3a4754616e7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b", + "cbe3ca627c6432c9a66f07c6411fccca894c9b083b55d0773152460142a676ed", + }, +}; + +static void test_keccak_hmac(const size_t * chunks) +{ + uint8_t inp_buff[1024]; + uint8_t key_buff[1024]; + uint8_t res_exp[32]; + uint8_t res_comp[32]; + const size_t len_chunks = chunks ? sizeof(chunks) / sizeof(*chunks) : 0; + + for (size_t i = 0; i < (sizeof(keccak_hmac_vectors) / sizeof(*keccak_hmac_vectors)); i++) + { + const size_t inp_len = strlen(keccak_hmac_vectors[i].inp)/2; + const size_t key_len = strlen(keccak_hmac_vectors[i].key)/2; + ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].inp, inp_len, inp_buff)); + ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].key, key_len, key_buff)); + ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].res, 32, res_exp)); + if (len_chunks == 0) + { + hmac_keccak_hash(res_comp, key_buff, key_len, inp_buff, inp_len); + } + else + { + hmac_keccak_state S; + hmac_keccak_init(&S, key_buff, key_len); + size_t inp_passed = 0; + size_t chunk_size = 0; + while(inp_passed < inp_len){ + const size_t to_pass = std::min(inp_len - inp_passed, chunks[chunk_size]); + hmac_keccak_update(&S, inp_buff + inp_passed, to_pass); + + inp_passed += to_pass; + chunk_size = (chunk_size + 1) % len_chunks; + } + + hmac_keccak_finish(&S, res_comp); + } + ASSERT_EQ(memcmp(res_exp, res_comp, 32), 0); + } +} + + +TEST(keccak_hmac, ) +{ + test_keccak_hmac({}); +} + +TEST(keccak_hmac, 1) +{ + static const size_t chunks[] = {1}; + test_keccak_hmac(chunks); +} + +TEST(keccak_hmac, 1_20) +{ + static const size_t chunks[] = {1, 20}; + test_keccak_hmac(chunks); +} + +TEST(keccak_hmac, 136_1) +{ + static const size_t chunks[] = {136, 1}; + test_keccak_hmac(chunks); +} + +TEST(keccak_hmac, 137_1) +{ + static const size_t chunks[] = {137, 1}; + test_keccak_hmac(chunks); +} diff --git a/tests/unit_tests/lmdb.cpp b/tests/unit_tests/lmdb.cpp new file mode 100644 index 000000000..c37c83a32 --- /dev/null +++ b/tests/unit_tests/lmdb.cpp @@ -0,0 +1,404 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/algorithm_ext/iota.hpp> +#include <boost/range/algorithm/equal.hpp> +#include <gtest/gtest.h> + +#include "lmdb/database.h" +#include "lmdb/table.h" +#include "lmdb/transaction.h" +#include "lmdb/util.h" + +namespace +{ + enum class choice : unsigned {}; + enum class big_choice : unsigned long {}; + + struct bytes { + char data[16]; + }; + + MONERO_CURSOR(test_cursor); + + template<typename T> + int run_compare(T left, T right, MDB_cmp_func* cmp) + { + MDB_val left_val = lmdb::to_val(left); + MDB_val right_val = lmdb::to_val(right); + return (*cmp)(&left_val, &right_val); + } +} + +TEST(LMDB, Traits) +{ + EXPECT_TRUE((std::is_same<void, lmdb::identity<void>::type>())); + EXPECT_TRUE((std::is_same<unsigned, lmdb::identity<unsigned>::type>())); + + EXPECT_TRUE((std::is_same<void, lmdb::native_type<void>>())); + EXPECT_TRUE((std::is_same<unsigned, lmdb::native_type<unsigned>>())); + EXPECT_TRUE((std::is_same<unsigned, lmdb::native_type<choice>>())); + EXPECT_TRUE((std::is_same<unsigned long, lmdb::native_type<big_choice>>())); +} + +TEST(LMDB, ToNative) +{ + enum class negative_choice : int {}; + + EXPECT_TRUE((std::is_same<unsigned, decltype(lmdb::to_native(choice(0)))>())); + EXPECT_TRUE( + (std::is_same<unsigned long, decltype(lmdb::to_native(big_choice(0)))>()) + ); + EXPECT_TRUE( + (std::is_same<int, decltype(lmdb::to_native(negative_choice(0)))>()) + ); + + EXPECT_EQ(unsigned(0), lmdb::to_native(choice(0))); + EXPECT_EQ(unsigned(0xffffffff), lmdb::to_native(choice(0xffffffff))); + EXPECT_EQ(-1, lmdb::to_native(negative_choice(-1))); + + // test constexpr + static_assert(100 == lmdb::to_native(choice(100)), "to_native failed"); + static_assert(-100 == lmdb::to_native(negative_choice(-100)), "to_native failed"); +} + +TEST(LMDB, Conversions) +{ + struct one + { + big_choice i; + choice j; + }; + + const one test{big_choice(100), choice(95)}; + one test2{big_choice(1000), choice(950)}; + + EXPECT_EQ(&test, lmdb::to_val(test).mv_data); + EXPECT_NE(&test2, lmdb::to_val(test).mv_data); + EXPECT_EQ( + &test, + static_cast<const void*>(lmdb::to_byte_span(lmdb::to_val(test)).begin()) + ); + EXPECT_EQ(sizeof(test), lmdb::to_val(test).mv_size); + EXPECT_EQ(sizeof(test), lmdb::to_byte_span(lmdb::to_val(test)).size()); + + EXPECT_EQ(&test2, lmdb::to_val(test2).mv_data); + EXPECT_NE(&test, lmdb::to_val(test2).mv_data); + EXPECT_EQ( + &test2, + static_cast<const void*>(lmdb::to_byte_span(lmdb::to_val(test2)).begin()) + ); + EXPECT_EQ(sizeof(test2), lmdb::to_val(test2).mv_size); + EXPECT_EQ(sizeof(test2), lmdb::to_byte_span(lmdb::to_val(test2)).size()); +} + +TEST(LMDB, LessSort) +{ + struct one + { + unsigned i; + unsigned j; + }; + + struct two + { + unsigned i; + choice j; + }; + + EXPECT_EQ(0, run_compare(0u, 0u, &lmdb::less<unsigned>)); + EXPECT_EQ(-1, run_compare(0u, 1u, &lmdb::less<unsigned>)); + EXPECT_EQ(1, run_compare(1u, 0u, &lmdb::less<unsigned>)); + + EXPECT_EQ(0, run_compare<one>({0, 1}, {0, 1}, &lmdb::less<unsigned, sizeof(unsigned)>)); + EXPECT_EQ(-1, run_compare<one>({0, 0}, {0, 1}, &lmdb::less<unsigned, sizeof(unsigned)>)); + EXPECT_EQ(1, run_compare<one>({0, 1}, {0, 0}, &lmdb::less<unsigned, sizeof(unsigned)>)); + + EXPECT_EQ(0, run_compare<one>({0, 1}, {0, 1}, MONERO_SORT_BY(one, j))); + EXPECT_EQ(-1, run_compare<one>({0, 0}, {0, 1}, MONERO_SORT_BY(one, j))); + EXPECT_EQ(1, run_compare<one>({0, 1}, {0, 0}, MONERO_SORT_BY(one, j))); + + EXPECT_EQ(0, run_compare<two>({0, choice(1)}, {0, choice(1)}, MONERO_SORT_BY(two, j))); + EXPECT_EQ(-1, run_compare<two>({0, choice(0)}, {0, choice(1)}, MONERO_SORT_BY(two, j))); + EXPECT_EQ(1, run_compare<two>({0, choice(1)}, {0, choice(0)}, MONERO_SORT_BY(two, j))); + + // compare function addresses + EXPECT_EQ((MONERO_SORT_BY(one, i)), (MONERO_SORT_BY(two, i))); + EXPECT_EQ((MONERO_SORT_BY(one, j)), (MONERO_SORT_BY(two, j))); + EXPECT_NE((MONERO_SORT_BY(one, i)), (MONERO_SORT_BY(two, j))); + EXPECT_NE((MONERO_SORT_BY(one, j)), (MONERO_SORT_BY(two, i))); +} + +TEST(LMDB, SortCompare) +{ + struct one + { + unsigned i; + bytes j; + }; + + one test{55}; + boost::iota(test.j.data, 10); + + const one test2 = test; + + EXPECT_EQ(0, run_compare(test, test2, MONERO_COMPARE(one, j))); + + test.j.data[15] = 1; + EXPECT_GT(0, run_compare(test, test2, MONERO_COMPARE(one, j))); + + test.j.data[15] = 100; + EXPECT_LT(0, run_compare(test, test2, MONERO_COMPARE(one, j))); +} + +TEST(LMDB, Table) +{ + struct one + { + bytes i; + bytes j; + }; + + constexpr lmdb::basic_table<choice, bytes> test{"foo"}; + + EXPECT_STREQ("foo", test.name); + static_assert(test.flags == 0, "bad flags"); + static_assert(&lmdb::less<unsigned> == test.key_cmp, "bad key_cmp"); + static_assert(test.value_cmp == nullptr, "bad value_cmp"); + EXPECT_TRUE(test.get_value<bytes>(MDB_val{}).matches(std::errc::invalid_argument)); + + lmdb::basic_table<big_choice, one> test2{ + "foo2", MDB_DUPSORT, &lmdb::compare<one> + }; + + EXPECT_STREQ("foo2", test2.name); + EXPECT_EQ((MDB_DUPSORT | MDB_DUPFIXED), test2.flags); + EXPECT_EQ(&lmdb::less<unsigned long>, test2.key_cmp); + EXPECT_EQ(&lmdb::compare<one>, test2.value_cmp); + EXPECT_TRUE(test2.get_value<one>(MDB_val{}).matches(std::errc::invalid_argument)); + + one record{}; + boost::iota(record.i.data, 0); + boost::iota(record.i.data, 20); + + const one record_copy = MONERO_UNWRAP(test2.get_value<one>(lmdb::to_val(record))); + EXPECT_TRUE(boost::equal(record.i.data, record_copy.i.data)); + EXPECT_TRUE(boost::equal(record.j.data, record_copy.j.data)); + + const bytes j_copy = MONERO_UNWRAP( + test2.get_value<MONERO_FIELD(one, j)>(lmdb::to_val(record)) + ); + EXPECT_TRUE(boost::equal(record.j.data, j_copy.data)); + + EXPECT_TRUE( + test.get_key_stream(test_cursor{}).matches(std::errc::invalid_argument) + ); + EXPECT_TRUE( + test2.get_key_stream(test_cursor{}).matches(std::errc::invalid_argument) + ); + + + EXPECT_TRUE( + test.get_value_stream(choice(0), test_cursor{}).matches(std::errc::invalid_argument) + ); + EXPECT_TRUE( + test2.get_value_stream(big_choice(0), test_cursor{}).matches(std::errc::invalid_argument) + ); +} + +TEST(LMDB, InvalidDatabase) +{ + lmdb::database test{lmdb::environment{}}; + + EXPECT_TRUE(test.resize().matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.create_read_txn().matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.reset_txn(lmdb::read_txn{}).matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.create_write_txn().matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.commit(lmdb::write_txn{}).matches(std::errc::invalid_argument)); + + EXPECT_TRUE( + test.try_write( [](MDB_txn&) { return success(); } ).matches(std::errc::invalid_argument) + ); +} + +TEST(LMDB, InvalidValueStream) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + lmdb::value_stream<one, close_test_cursor> test{test_cursor{}}; + + EXPECT_TRUE((std::is_same<one, decltype(*(test.make_iterator()))>())); + EXPECT_TRUE((std::is_same<one, decltype(*(test.make_range().begin()))>())); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test.make_iterator<MONERO_FIELD(one, k)>()))>()) + ); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test.make_range<MONERO_FIELD(one, k)>().begin()))>()) + ); + + EXPECT_NO_THROW(test.reset()); + EXPECT_EQ(0u, test.count()); + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); + + EXPECT_EQ(0u, test.count()); + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); +} + +TEST(LMDB, InvalidValueIterator) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + lmdb::value_iterator<one> test1{}; + + EXPECT_TRUE((std::is_same<one, decltype(*test1)>())); + EXPECT_TRUE( + (std::is_same<bytes, decltype(test1.get_value<MONERO_FIELD(one, k)>())>()) + ); + + EXPECT_TRUE(test1.is_end()); + EXPECT_NO_THROW(++test1); + EXPECT_NO_THROW(test1++); + EXPECT_TRUE(test1.is_end()); + + lmdb::value_iterator<one> test2{nullptr}; + + EXPECT_TRUE(test2.is_end()); + EXPECT_NO_THROW(++test2); + EXPECT_NO_THROW(test2++); + EXPECT_TRUE(test2.is_end()); + + EXPECT_TRUE(test1.equal(test2)); + EXPECT_TRUE(test2.equal(test1)); + EXPECT_TRUE(test1 == test2); + EXPECT_TRUE(test2 == test1); + EXPECT_FALSE(test1 != test2); + EXPECT_FALSE(test2 != test1); + + lmdb::value_iterator<MONERO_FIELD(one, k)> test3{}; + + EXPECT_TRUE((std::is_same<bytes, decltype(*test3)>())); + EXPECT_TRUE((std::is_same<one, decltype(test3.get_value<one>())>())); + EXPECT_TRUE( + (std::is_same<choice, decltype(test1.get_value<MONERO_FIELD(one, j)>())>()) + ); + + EXPECT_TRUE(test3.is_end()); + EXPECT_NO_THROW(++test3); + EXPECT_NO_THROW(test3++); + EXPECT_TRUE(test3.is_end()); +} + +TEST(LMDB, InvalidKeyStream) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + using record = std::pair<choice, boost::iterator_range<lmdb::value_iterator<one>>>; + + lmdb::key_stream<choice, one, close_test_cursor> test{test_cursor{}}; + + EXPECT_TRUE((std::is_same<record, decltype(*(test.make_iterator()))>())); + EXPECT_TRUE((std::is_same<record, decltype(*(test.make_range().begin()))>())); + + EXPECT_NO_THROW(test.reset()); + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); + + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); +} + +TEST(LMDB, InvalidKeyIterator) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + using record = std::pair<choice, boost::iterator_range<lmdb::value_iterator<one>>>; + + lmdb::key_iterator<choice, one> test1{}; + + EXPECT_TRUE((std::is_same<record, decltype(*test1)>())); + EXPECT_TRUE((std::is_same<choice, decltype(test1.get_key())>())); + EXPECT_TRUE((std::is_same<one, decltype(*(test1.make_value_iterator()))>())); + EXPECT_TRUE((std::is_same<one, decltype(*(test1.make_value_range().begin()))>())); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test1.make_value_iterator<MONERO_FIELD(one, k)>()))>()) + ); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test1.make_value_range<MONERO_FIELD(one, k)>().begin()))>()) + ); + + EXPECT_TRUE(test1.is_end()); + EXPECT_NO_THROW(++test1); + EXPECT_NO_THROW(test1++); + EXPECT_TRUE(test1.is_end()); + EXPECT_TRUE(test1.make_value_iterator().is_end()); + EXPECT_TRUE(test1.make_value_range().empty()); + + lmdb::key_iterator<choice, one> test2{nullptr}; + + EXPECT_TRUE(test2.is_end()); + EXPECT_NO_THROW(++test2); + EXPECT_NO_THROW(test2++); + EXPECT_TRUE(test2.is_end()); + EXPECT_TRUE(test2.make_value_iterator().is_end()); + EXPECT_TRUE(test2.make_value_range().empty()); + + EXPECT_TRUE(test1.equal(test2)); + EXPECT_TRUE(test2.equal(test1)); + EXPECT_TRUE(test1 == test2); + EXPECT_TRUE(test2 == test1); + EXPECT_FALSE(test1 != test2); + EXPECT_FALSE(test2 != test1); +} + + diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp index f37b608b6..ce6ff3551 100644 --- a/tests/unit_tests/long_term_block_weight.cpp +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -64,6 +64,18 @@ public: virtual uint64_t height() const override { return blocks.size(); } virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight); + return ret; + } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight); + return ret; + } virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override { uint64_t h = height(); crypto::hash top = crypto::null_hash; diff --git a/tests/unit_tests/multisig.cpp b/tests/unit_tests/multisig.cpp index 9a74b3dce..c5917200e 100644 --- a/tests/unit_tests/multisig.cpp +++ b/tests/unit_tests/multisig.cpp @@ -71,7 +71,7 @@ static void make_wallet(unsigned int idx, tools::wallet2 &wallet) try { - wallet.init(""); + wallet.init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet.set_subaddress_lookahead(1, 1); wallet.generate("", "", spendkey, true, false); ASSERT_TRUE(test_addresses[idx].address == wallet.get_account().get_public_address_str(cryptonote::TESTNET)); diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index a38ecfe81..77fb71d96 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -33,6 +33,7 @@ #include <boost/asio/io_service.hpp> #include <boost/asio/ip/tcp.hpp> #include <boost/asio/read.hpp> +#include <boost/asio/steady_timer.hpp> #include <boost/asio/write.hpp> #include <boost/endian/conversion.hpp> #include <boost/system/error_code.hpp> @@ -45,6 +46,7 @@ #include "net/error.h" #include "net/net_utils_base.h" #include "net/socks.h" +#include "net/socks_connect.h" #include "net/parse.h" #include "net/tor_address.h" #include "p2p/net_peerlist_boost_serialization.h" @@ -742,4 +744,92 @@ TEST(socks_client, resolve_command) while (test_client->called_ == 1); } +TEST(socks_connector, host) +{ + io_thread io{}; + boost::asio::steady_timer timeout{io.io_service}; + timeout.expires_from_now(std::chrono::seconds{5}); + + boost::unique_future<boost::asio::ip::tcp::socket> sock = + net::socks::connector{io.acceptor.local_endpoint()}("example.com", "8080", timeout); + + while (!io.connected); + const std::uint8_t expected_bytes[] = { + 4, 1, 0x1f, 0x90, 0x00, 0x00, 0x00, 0x01, 0x00, + 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', 0x00 + }; + + std::uint8_t actual_bytes[sizeof(expected_bytes)]; + boost::asio::read(io.server, boost::asio::buffer(actual_bytes)); + EXPECT_TRUE(std::memcmp(expected_bytes, actual_bytes, sizeof(actual_bytes)) == 0); + + const std::uint8_t reply_bytes[] = {0, 90, 0, 0, 0, 0, 0, 0}; + boost::asio::write(io.server, boost::asio::buffer(reply_bytes)); + + ASSERT_EQ(boost::future_status::ready, sock.wait_for(boost::chrono::seconds{3})); + EXPECT_TRUE(sock.get().is_open()); +} + +TEST(socks_connector, ipv4) +{ + io_thread io{}; + boost::asio::steady_timer timeout{io.io_service}; + timeout.expires_from_now(std::chrono::seconds{5}); + + boost::unique_future<boost::asio::ip::tcp::socket> sock = + net::socks::connector{io.acceptor.local_endpoint()}("250.88.125.99", "8080", timeout); + + while (!io.connected); + const std::uint8_t expected_bytes[] = { + 4, 1, 0x1f, 0x90, 0xfa, 0x58, 0x7d, 0x63, 0x00 + }; + + std::uint8_t actual_bytes[sizeof(expected_bytes)]; + boost::asio::read(io.server, boost::asio::buffer(actual_bytes)); + EXPECT_TRUE(std::memcmp(expected_bytes, actual_bytes, sizeof(actual_bytes)) == 0); + + const std::uint8_t reply_bytes[] = {0, 90, 0, 0, 0, 0, 0, 0}; + boost::asio::write(io.server, boost::asio::buffer(reply_bytes)); + + ASSERT_EQ(boost::future_status::ready, sock.wait_for(boost::chrono::seconds{3})); + EXPECT_TRUE(sock.get().is_open()); +} + +TEST(socks_connector, error) +{ + io_thread io{}; + boost::asio::steady_timer timeout{io.io_service}; + timeout.expires_from_now(std::chrono::seconds{5}); + + boost::unique_future<boost::asio::ip::tcp::socket> sock = + net::socks::connector{io.acceptor.local_endpoint()}("250.88.125.99", "8080", timeout); + + while (!io.connected); + const std::uint8_t expected_bytes[] = { + 4, 1, 0x1f, 0x90, 0xfa, 0x58, 0x7d, 0x63, 0x00 + }; + + std::uint8_t actual_bytes[sizeof(expected_bytes)]; + boost::asio::read(io.server, boost::asio::buffer(actual_bytes)); + EXPECT_TRUE(std::memcmp(expected_bytes, actual_bytes, sizeof(actual_bytes)) == 0); + + const std::uint8_t reply_bytes[] = {0, 91, 0, 0, 0, 0, 0, 0}; + boost::asio::write(io.server, boost::asio::buffer(reply_bytes)); + + ASSERT_EQ(boost::future_status::ready, sock.wait_for(boost::chrono::seconds{3})); + EXPECT_THROW(sock.get().is_open(), boost::system::system_error); +} + +TEST(socks_connector, timeout) +{ + io_thread io{}; + boost::asio::steady_timer timeout{io.io_service}; + timeout.expires_from_now(std::chrono::milliseconds{10}); + + boost::unique_future<boost::asio::ip::tcp::socket> sock = + net::socks::connector{io.acceptor.local_endpoint()}("250.88.125.99", "8080", timeout); + + ASSERT_EQ(boost::future_status::ready, sock.wait_for(boost::chrono::seconds{3})); + EXPECT_THROW(sock.get().is_open(), boost::system::system_error); +} diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index eb70caefc..27b14ffff 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -1187,3 +1187,26 @@ TEST(Serialization, portability_signed_tx) ASSERT_TRUE(epee::string_tools::pod_to_hex(ki1) == "d54cbd435a8d636ad9b01b8d4f3eb13bd0cf1ce98eddf53ab1617f9b763e66c0"); ASSERT_TRUE(epee::string_tools::pod_to_hex(ki2) == "6c3cd6af97c4070a7aef9b1344e7463e29c7cd245076fdb65da447a34da3ca76"); } + +TEST(Serialization, difficulty_type) +{ + std::vector<cryptonote::difficulty_type> v_original; + + for(int i = 0; i != 100; i++) + { + v_original.push_back(cryptonote::difficulty_type("117868131154734361989189100")); + if(v_original.size() > 1) + v_original.back() *= v_original[v_original.size()-2]; + } + + std::stringstream ss; + boost::archive::portable_binary_oarchive a(ss); + a << v_original; + + std::vector<cryptonote::difficulty_type> v_unserialized; + + boost::archive::portable_binary_iarchive a2(ss); + a2 >> v_unserialized; + + ASSERT_EQ(v_original, v_unserialized); +} |