diff options
72 files changed, 5347 insertions, 574 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index b2d47706b..543e58eca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -887,10 +887,16 @@ find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_ set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES}) if(NOT Boost_FOUND) - die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (1.58) or the equivalent") + die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (>=1.58) or the equivalent") elseif(Boost_FOUND) message(STATUS "Found Boost Version: ${Boost_VERSION}") - if (Boost_VERSION VERSION_LESS 106200 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + if (Boost_VERSION VERSION_LESS 10 AND Boost_VERSION VERSION_LESS 1.62.0 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + set(BOOST_BEFORE_1_62 true) + endif() + if (NOT Boost_VERSION VERSION_LESS 10 AND Boost_VERSION VERSION_LESS 106200 AND NOT (OPENSSL_VERSION VERSION_LESS 1.1)) + set(BOOST_BEFORE_1_62 true) + endif() + if (BOOST_BEFORE_1_62) message(FATAL_ERROR "Boost older than 1.62 is too old to link with OpenSSL 1.1 or newer. " "Update Boost or install OpenSSL 1.0 and set path to it when running cmake: " "cmake -DOPENSSL_ROOT_DIR='/usr/include/openssl-1.0;/usr/lib/openssl-1.0'") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 883d366aa..7b184c00a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,26 +88,26 @@ C4 is meant to provide a reusable optimal collaboration model for open source so ### Preliminaries -- The project SHALL use the git distributed revision control system. -- The project SHALL be hosted on github.com or equivalent, herein called the "Platform". -- The project SHALL use the Platform issue tracker. +- The project MUST use the git distributed revision control system. +- The project MUST be hosted on github.com or equivalent, herein called the "Platform". +- The project MUST use the Platform issue tracker. - Non-GitHub example: - "Platform" could be a vanilla git repo and Trac hosted on the same machine/network. - The Platform issue tracker would be Trac. - The project SHOULD have clearly documented guidelines for code style. - A "Contributor" is a person who wishes to provide a patch, being a set of commits that solve some clearly identified problem. - A "Maintainer" is a person who merges patches to the project. Maintainers are not developers; their job is to enforce process. -- Contributors SHALL NOT have commit access to the repository unless they are also Maintainers. -- Maintainers SHALL have commit access to the repository. -- Everyone, without distinction or discrimination, SHALL have an equal right to become a Contributor under the terms of this contract. +- Contributors MUST NOT have commit access to the repository unless they are also Maintainers. +- Maintainers MUST have commit access to the repository. +- Everyone, without distinction or discrimination, MUST have an equal right to become a Contributor under the terms of this contract. ### Licensing and ownership -- The project SHALL use a share-alike license, such as BSD-3, the GPLv3 or a variant thereof (LGPL, AGPL), or the MPLv2. -- All contributions to the project source code ("patches") SHALL use the same license as the project. -- All patches are owned by their authors. There SHALL NOT be any copyright assignment process. -- The copyrights in the project SHALL be owned collectively by all its Contributors. -- Each Contributor SHALL be responsible for identifying themselves in the project Contributor list. +- The project MUST use a share-alike license, such as BSD-3, the GPLv3 or a variant thereof (LGPL, AGPL), or the MPLv2. +- All contributions to the project source code ("patches") MUST use the same license as the project. +- All patches are owned by their authors. There MUST NOT be any copyright assignment process. +- The copyrights in the project MUST be owned collectively by all its Contributors. +- Each Contributor MUST be responsible for identifying themselves in the project Contributor list. ### Patch requirements @@ -116,27 +116,27 @@ C4 is meant to provide a reusable optimal collaboration model for open source so - A patch SHOULD be a minimal and accurate answer to exactly one identified and agreed problem. - A patch MUST adhere to the code style guidelines of the project if these are defined. - A patch MUST adhere to the "Evolution of Public Contracts" guidelines defined below. -- A patch SHALL NOT include non-trivial code from other projects unless the Contributor is the original author of that code. +- A patch MUST NOT include non-trivial code from other projects unless the Contributor is the original author of that code. - A patch MUST compile cleanly and pass project self-tests on at least the principle target platform. - A patch commit message SHOULD consist of a single short (less than 50 character) line summarizing the change, optionally followed by a blank line and then a more thorough description. - A "Correct Patch" is one that satisfies the above requirements. ### Development process -- Change on the project SHALL be governed by the pattern of accurately identifying problems and applying minimal, accurate solutions to these problems. +- Change on the project MUST be governed by the pattern of accurately identifying problems and applying minimal, accurate solutions to these problems. - To request changes, a user SHOULD log an issue on the project Platform issue tracker. - The user or Contributor SHOULD write the issue by describing the problem they face or observe. - The user or Contributor SHOULD seek consensus on the accuracy of their observation, and the value of solving the problem. -- Users SHALL NOT log feature requests, ideas, or suggestions unrelated to Monero code or Monero's dependency code or Monero's potential/future dependency code or research which successfully implements Monero. -- Users SHALL NOT log any solutions to problems (verifiable or hypothetical) of which are not explicitly documented and/or not provable and/or cannot be reasonably proven. -- Thus, the release history of the project SHALL be a list of meaningful issues logged and solved. -- To work on an issue, a Contributor SHALL fork the project repository and then work on their forked repository. -- To submit a patch, a Contributor SHALL create a Platform pull request back to the project. -- A Contributor SHALL NOT commit changes directly to the project. +- Users MUST NOT log feature requests, ideas, or suggestions unrelated to Monero code or Monero's dependency code or Monero's potential/future dependency code or research which successfully implements Monero. +- Users MUST NOT log any solutions to problems (verifiable or hypothetical) of which are not explicitly documented and/or not provable and/or cannot be reasonably proven. +- Thus, the release history of the project MUST be a list of meaningful issues logged and solved. +- To work on an issue, a Contributor MUST fork the project repository and then work on their forked repository. +- To submit a patch, a Contributor MUST create a Platform pull request back to the project. +- A Contributor MUST NOT commit changes directly to the project. - To discuss a patch, people MAY comment on the Platform pull request, on the commit, or elsewhere. -- To accept or reject a patch, a Maintainer SHALL use the Platform interface. +- To accept or reject a patch, a Maintainer MUST use the Platform interface. - Maintainers SHOULD NOT merge their own patches except in exceptional cases, such as non-responsiveness from other Maintainers for an extended period (more than 30 days) or unless urgent as defined by the Monero Maintainers Team. -- Maintainers SHALL NOT make value judgments on correct patches unless the Maintainer (as may happen in rare circumstances) is a core code developer. +- Maintainers MUST NOT make value judgments on correct patches unless the Maintainer (as may happen in rare circumstances) is a core code developer. - Maintainers MUST NOT merge pull requests in less than 168 hours (1 week) unless deemed urgent by at least 2 people from the Monero Maintainer Team. - The Contributor MAY tag an issue as "Ready" after making a pull request for the issue. - The user who created an issue SHOULD close the issue after checking the patch is successful. @@ -146,27 +146,27 @@ C4 is meant to provide a reusable optimal collaboration model for open source so ### Creating stable releases -- The project SHALL have one branch ("master") that always holds the latest in-progress version and SHOULD always build. -- The project SHALL NOT use topic branches for any reason. Personal forks MAY use topic branches. -- To make a stable release someone SHALL fork the repository by copying it and thus become maintainer of this repository. +- The project MUST have one branch ("master") that always holds the latest in-progress version and SHOULD always build. +- The project MUST NOT use topic branches for any reason. Personal forks MAY use topic branches. +- To make a stable release someone MUST fork the repository by copying it and thus become maintainer of this repository. - Forking a project for stabilization MAY be done unilaterally and without agreement of project maintainers. -- A patch to a stabilization project declared "stable" SHALL be accompanied by a reproducible test case. +- A patch to a stabilization project declared "stable" MUST be accompanied by a reproducible test case. ### Evolution of public contracts -- All Public Contracts (APIs or protocols) SHALL be documented. +- All Public Contracts (APIs or protocols) MUST be documented. - All Public Contracts SHOULD have space for extensibility and experimentation. - A patch that modifies a stable Public Contract SHOULD not break existing applications unless there is overriding consensus on the value of doing this. - A patch that introduces new features to a Public Contract SHOULD do so using new names. - Old names SHOULD be deprecated in a systematic fashion by marking new names as "experimental" until they are stable, then marking the old names as "deprecated". - When sufficient time has passed, old deprecated names SHOULD be marked "legacy" and eventually removed. -- Old names SHALL NOT be reused by new features. +- Old names MUST NOT be reused by new features. - When old names are removed, their implementations MUST provoke an exception (assertion) if used by applications. ### Project administration -- The project founders SHALL act as Administrators to manage the set of project Maintainers. -- The Administrators SHALL ensure their own succession over time by promoting the most effective Maintainers. -- A new Contributor who makes a correct patch SHALL be invited to become a Maintainer. +- The project founders MUST act as Administrators to manage the set of project Maintainers. +- The Administrators MUST ensure their own succession over time by promoting the most effective Maintainers. +- A new Contributor who makes a correct patch MUST be invited to become a Maintainer. - Administrators MAY remove Maintainers who are inactive for an extended period of time, or who repeatedly fail to apply this process accurately. - Administrators SHOULD block or ban "bad actors" who cause stress and pain to others in the project. This should be done after public discussion, with a chance for all parties to speak. A bad actor is someone who repeatedly ignores the rules and culture of the project, who is needlessly argumentative or hostile, or who is offensive, and who is unable to self-correct their behavior when asked to do so by others. diff --git a/Dockerfile b/Dockerfile index 9fe7cfb8f..d932e0173 100644 --- a/Dockerfile +++ b/Dockerfile @@ -185,8 +185,14 @@ RUN set -ex && \ rm -rf /var/lib/apt COPY --from=builder /src/build/release/bin /usr/local/bin/ +# Create monero user +RUN adduser --system --group --disabled-password monero && \ + mkdir -p /wallet /home/monero/.bitmonero && \ + chown -R monero:monero /home/monero/.bitmonero && \ + chown -R monero:monero /wallet + # Contains the blockchain -VOLUME /root/.bitmonero +VOLUME /home/monero/.bitmonero # Generate your wallet via accessing the container and run: # cd /wallet @@ -196,5 +202,8 @@ VOLUME /wallet EXPOSE 18080 EXPOSE 18081 +# switch to user monero +USER monero + ENTRYPOINT ["monerod", "--p2p-bind-ip=0.0.0.0", "--p2p-bind-port=18080", "--rpc-bind-ip=0.0.0.0", "--rpc-bind-port=18081", "--non-interactive", "--confirm-external-bind"] @@ -73,6 +73,8 @@ Monero is a private, secure, untraceable, decentralised digital currency. You ar **Untraceability:** By taking advantage of ring signatures, a special property of a certain type of cryptography, Monero is able to ensure that transactions are not only untraceable, but have an optional measure of ambiguity that ensures that transactions cannot easily be tied back to an individual user or computer. +**Decentralization:** The utility of monero depends on its decentralised peer-to-peer consensus network - anyone should be able to run the monero software, validate the integrity of the blockchain, and participate in all aspects of the monero network using consumer-grade commodity hardware. Decentralization of the monero network is maintained by software development that minimizes the costs of running the monero software and inhibits the proliferation of specialized, non-commodity hardware. + ## About this project This is the core implementation of Monero. It is open source and completely free to use without restrictions, except for those specified in the license agreement below. There are no restrictions on anyone creating an alternative implementation of Monero that uses the protocol and network in a compatible manner. diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index f5548c585..e4b504466 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -58,6 +58,12 @@ #define DEFAULT_TIMEOUT_MS_REMOTE 300000 // 5 minutes #define TIMEOUT_EXTRA_MS_PER_BYTE 0.2 +#if BOOST_VERSION >= 107000 +#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context()) +#else +#define GET_IO_SERVICE(s) ((s).get_io_service()) +#endif + PRAGMA_WARNING_PUSH namespace epee { @@ -99,7 +105,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_connection_type( connection_type ), m_throttle_speed_in("speed_in", "throttle_speed_in"), m_throttle_speed_out("speed_out", "throttle_speed_out"), - m_timer(socket_.get_io_service()), + m_timer(GET_IO_SERVICE(socket_)), m_local(false), m_ready_to_close(false) { @@ -243,7 +249,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> boost::asio::io_service& connection<t_protocol_handler>::get_io_service() { - return socket().get_io_service(); + return GET_IO_SERVICE(socket()); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -487,7 +493,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(!m_is_multithreaded) { //single thread model, we can wait in blocked call - size_t cnt = socket().get_io_service().run_one(); + size_t cnt = GET_IO_SERVICE(socket()).run_one(); if(!cnt)//service is going to quit return false; }else @@ -497,7 +503,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //if no handlers were called //TODO: Maybe we need to have have critical section + event + callback to upper protocol to //ask it inside(!) critical region if we still able to go in event wait... - size_t cnt = socket().get_io_service().poll_one(); + size_t cnt = GET_IO_SERVICE(socket()).poll_one(); if(!cnt) misc_utils::sleep_no_w(1); } @@ -670,9 +676,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size"); reset_timer(get_default_timeout(), false); async_write(boost::asio::buffer(m_send_que.front().data(), size_now ) , - //strand_.wrap( + strand_.wrap( boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) - //) + ) ); //_dbg3("(chunk): " << size_now); //logger_handle_net_write(size_now); @@ -761,8 +767,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) // Initiate graceful connection closure. m_timer.cancel(); boost::system::error_code ignored_ec; + if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) + socket_.shutdown(ignored_ec); socket().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); - socket().close(); if (!m_host.empty()) { try { host_count(m_host, -1); } catch (...) { /* ignore */ } @@ -855,9 +862,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H))) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), void(), "Unexpected queue size"); async_write(boost::asio::buffer(m_send_que.front().data(), size_now) , - // strand_.wrap( + strand_.wrap( boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2) - // ) + ) ); //_dbg3("(normal)" << size_now); } @@ -1207,7 +1214,7 @@ POP_WARNINGS template<class t_protocol_handler> bool boosted_tcp_server<t_protocol_handler>::add_connection(t_connection_context& out, boost::asio::ip::tcp::socket&& sock, network_address real_remote, epee::net_utils::ssl_support_t ssl_support) { - if(std::addressof(get_io_service()) == std::addressof(sock.get_io_service())) + if(std::addressof(get_io_service()) == std::addressof(GET_IO_SERVICE(sock))) { connection_ptr conn(new connection<t_protocol_handler>(std::move(sock), m_state, m_connection_type, ssl_support, m_ssl_context)); if(conn->start(false, 1 < m_threads_count, std::move(real_remote))) diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index 83db171d8..cafc08ead 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -47,6 +47,12 @@ // TODO: #include "net/network_throttle-detail.hpp" +#if BOOST_VERSION >= 107000 +#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context()) +#else +#define GET_IO_SERVICE(s) ((s).get_io_service()) +#endif + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.conn" @@ -117,8 +123,8 @@ connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::s : m_stats(std::move(stats)), mI( new connection_basic_pimpl("peer") ), - strand_(sock.get_io_service()), - socket_(sock.get_io_service(), ssl_context.context), + strand_(GET_IO_SERVICE(sock)), + socket_(GET_IO_SERVICE(sock), ssl_context.context), m_want_close_connection(false), m_was_shutdown(false), m_ssl_support(ssl_support), @@ -167,7 +173,6 @@ connection_basic::~connection_basic() noexcept(false) { std::string remote_addr_str = "?"; try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ; _note("Destructing connection #"<<mI->m_peer_number << " to " << remote_addr_str); -try { throw 0; } catch(...){} } void connection_basic::set_rate_up_limit(uint64_t limit) { diff --git a/external/trezor-common b/external/trezor-common -Subproject 588f8e03f5ac111adf719f0a437de67481a26ae +Subproject cb238cb1f134accc4200217d9511115a8f61c6c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2ae72ee81..da6d76d97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,6 +109,7 @@ add_subdirectory(ringct) add_subdirectory(checkpoints) add_subdirectory(cryptonote_basic) add_subdirectory(cryptonote_core) +add_subdirectory(lmdb) add_subdirectory(multisig) add_subdirectory(net) if(NOT IOS) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 313e716e0..d2fe39fc2 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -947,6 +947,17 @@ public: virtual size_t get_block_weight(const uint64_t& height) const = 0; /** + * @brief fetch the last N blocks' weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const = 0; + + /** * @brief fetch a block's cumulative difficulty * * The subclass should return the cumulative difficulty of the block with the @@ -1000,6 +1011,17 @@ public: virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; /** + * @brief fetch the last N blocks' long term weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const = 0; + + /** * @brief fetch a block's hash * * The subclass should return hash of the block with the diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index becffed16..3391d9bff 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2445,6 +2445,70 @@ size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const return ret; } +std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + const uint64_t h = height(); + if (start_height >= h) + throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str())); + + std::vector<uint64_t> ret; + ret.reserve(count); + + MDB_val v; + uint64_t range_begin = 0, range_end = 0; + for (uint64_t height = start_height; height < h && count--; ++height) + { + if (height >= range_begin && height < range_end) + { + // nothing to do + } + else + { + int result = 0; + if (range_end > 0) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str())); + } + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); + ret.push_back(*(const uint64_t*)(((const char*)bi) + offset)); + } + + TXN_POSTFIX_RDONLY(); + return ret; +} + +std::vector<uint64_t> BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight)); +} + +std::vector<uint64_t> BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight)); +} + difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 7a04c5f0c..9185bd409 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -219,6 +219,8 @@ public: virtual size_t get_block_weight(const uint64_t& height) const; + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const; + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; virtual difficulty_type get_block_difficulty(const uint64_t& height) const; @@ -227,6 +229,8 @@ public: virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; @@ -394,6 +398,8 @@ private: virtual uint64_t get_database_size() const; + std::vector<uint64_t> get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const; + // fix up anything that may be wrong due to past bugs virtual void fixup(); diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index eb9c3e45a..7916364c5 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -72,10 +72,12 @@ public: virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; } virtual uint64_t get_top_block_timestamp() const { return 0; } virtual size_t get_block_weight(const uint64_t& height) const { return 128; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const { return {}; } virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; } virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; } virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; } virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const { return {}; } virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<cryptonote::block>(); } virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<crypto::hash>(); } diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 84f002929..d22d59b36 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -39,6 +39,7 @@ set(crypto_sources hash-extra-jh.c hash-extra-skein.c hash.c + hmac-keccak.c jh.c keccak.c oaes_lib.c @@ -64,6 +65,7 @@ set(crypto_private_headers groestl_tables.h hash-ops.h hash.h + hmac-keccak.h initializer.h jh.h keccak.h diff --git a/src/crypto/CryptonightR_JIT.c b/src/crypto/CryptonightR_JIT.c index 68258a959..ee8f3f36f 100644 --- a/src/crypto/CryptonightR_JIT.c +++ b/src/crypto/CryptonightR_JIT.c @@ -4,6 +4,9 @@ #include <string.h> #include <stdio.h> #include <unistd.h> +#if !(defined(_MSC_VER) || defined(__MINGW32__)) +#include <sys/mman.h> +#endif #include "int-util.h" #include "hash-ops.h" @@ -58,6 +61,11 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f uint8_t* JIT_code = (uint8_t*) buf; const uint8_t* JIT_code_end = JIT_code + buf_size; +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_WRITE)) + return 1; +#endif + APPEND_CODE(prologue, sizeof(prologue)); uint32_t prev_rot_src = 0xFFFFFFFFU; @@ -101,6 +109,11 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f APPEND_CODE(epilogue, sizeof(epilogue)); +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_EXEC)) + return 1; +#endif + __builtin___clear_cache((char*)buf, (char*)JIT_code); return 0; diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index efdeef8d1..6d4905ad5 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -33,11 +33,11 @@ extern "C" #define RC_LENGTH (5 * (AES_BLOCK_SIZE / 4 - 2)) #if defined(_MSC_VER) -#define ALIGN __declspec(align(TABLE_ALIGN)) +#define LOCAL_ALIGN __declspec(align(TABLE_ALIGN)) #elif defined(__GNUC__) -#define ALIGN __attribute__ ((aligned(16))) +#define LOCAL_ALIGN __attribute__ ((aligned(16))) #else -#define ALIGN +#define LOCAL_ALIGN #endif #define rf1(r,c) (r) @@ -131,7 +131,7 @@ extern "C" #define t_set(m,n) t_##m##n #define t_use(m,n) t_##m##n -#define d_4(t,n,b,e,f,g,h) ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } +#define d_4(t,n,b,e,f,g,h) LOCAL_ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } #define four_tables(x,tab,vf,rf,c) \ (tab[0][bval(vf(x,0,c),rf(0,c))] \ diff --git a/src/crypto/hmac-keccak.c b/src/crypto/hmac-keccak.c new file mode 100644 index 000000000..edcb2065e --- /dev/null +++ b/src/crypto/hmac-keccak.c @@ -0,0 +1,81 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "hmac-keccak.h" +#include "memwipe.h" + +#define KECCAK_BLOCKLEN 136 +#define HASH_SIZE 32 + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen) { + const uint8_t *key = _key; + uint8_t keyhash[HASH_SIZE]; + uint8_t pad[KECCAK_BLOCKLEN]; + uint64_t i; + + if (keylen > KECCAK_BLOCKLEN) { + keccak(key, keylen, keyhash, HASH_SIZE); + key = keyhash; + keylen = HASH_SIZE; + } + + keccak_init(&S->inner); + memset(pad, 0x36, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->inner, pad, KECCAK_BLOCKLEN); + + keccak_init(&S->outer); + memset(pad, 0x5c, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->outer, pad, KECCAK_BLOCKLEN); + + memwipe(keyhash, HASH_SIZE); +} + +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen) { + keccak_update(&S->inner, data, datalen); +} + +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest) { + uint8_t ihash[HASH_SIZE]; + keccak_finish(&S->inner, ihash); + keccak_update(&S->outer, ihash, HASH_SIZE); + keccak_finish(&S->outer, digest); + memwipe(ihash, HASH_SIZE); +} + +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen) { + hmac_keccak_state S; + hmac_keccak_init(&S, key, keylen); + hmac_keccak_update(&S, in, inlen); + hmac_keccak_finish(&S, out); +} diff --git a/src/crypto/hmac-keccak.h b/src/crypto/hmac-keccak.h new file mode 100644 index 000000000..c450860d4 --- /dev/null +++ b/src/crypto/hmac-keccak.h @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef HMAC_KECCAK_H +#define HMAC_KECCAK_H + +#include "keccak.h" + +// HMAC RFC 2104 with Keccak-256 base hash function +// +// B = KECCAK_BLOCKLEN = 136 B +// L = HASH_SIZE = 32 B +// +// Note this is not HMAC-SHA3 as SHA3 and Keccak differs in +// the padding constant. + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + KECCAK_CTX inner; + KECCAK_CTX outer; +} hmac_keccak_state; + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen); +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen); +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest); +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen); + +#ifdef __cplusplus +} +#endif +#endif //HMAC_KECCAK_H diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 9dc6f5038..7f36c9dc3 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -76,7 +76,7 @@ static inline int use_v4_jit(void) const char *env = getenv("MONERO_USE_CNV4_JIT"); if (!env) { - use_v4_jit_flag = 0; + use_v4_jit_flag = 1; } else if (!strcmp(env, "0") || !strcmp(env, "no")) { use_v4_jit_flag = 0; @@ -274,10 +274,10 @@ static inline int use_v4_jit(void) #define VARIANT2_2() \ do if (variant == 2 || variant == 3) \ { \ - *U64(hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ - *(U64(hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ - hi ^= SWAP64LE(*U64(hp_state + (j ^ 0x20))); \ - lo ^= SWAP64LE(*(U64(hp_state + (j ^ 0x20)) + 1)); \ + *U64(local_hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ + *(U64(local_hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ + hi ^= SWAP64LE(*U64(local_hp_state + (j ^ 0x20))); \ + lo ^= SWAP64LE(*(U64(local_hp_state + (j ^ 0x20)) + 1)); \ } while (0) #define V4_REG_LOAD(dst, src) \ @@ -405,7 +405,7 @@ static inline int use_v4_jit(void) #define pre_aes() \ j = state_index(a); \ - _c = _mm_load_si128(R128(&hp_state[j])); \ + _c = _mm_load_si128(R128(&local_hp_state[j])); \ _a = _mm_load_si128(R128(a)); \ /* @@ -418,20 +418,20 @@ static inline int use_v4_jit(void) * This code is based upon an optimized implementation by dga. */ #define post_aes() \ - VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ - VARIANT1_1(&hp_state[j]); \ + _mm_store_si128(R128(&local_hp_state[j]), _mm_xor_si128(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_INTEGER_MATH_SSE2(b, c); \ VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ - VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ @@ -756,10 +756,10 @@ void slow_hash_allocate_state(void) #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, 0, 0); + MAP_PRIVATE | MAP_ANON, -1, 0); #else hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 0, 0); + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); #endif if(hp_state == MAP_FAILED) hp_state = NULL; @@ -778,11 +778,16 @@ void slow_hash_allocate_state(void) #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ defined(__DragonFly__) || defined(__NetBSD__) - hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANON, 0, 0); +#ifdef __NetBSD__ +#define RESERVED_FLAGS PROT_MPROTECT(PROT_EXEC) +#else +#define RESERVED_FLAGS 0 +#endif + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | RESERVED_FLAGS, + MAP_PRIVATE | MAP_ANON, -1, 0); #else - hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); #endif if(hp_jitfunc_memory == MAP_FAILED) hp_jitfunc_memory = NULL; @@ -794,9 +799,6 @@ void slow_hash_allocate_state(void) hp_jitfunc_memory = malloc(4096 + 4095); } hp_jitfunc = (v4_random_math_JIT_func)((size_t)(hp_jitfunc_memory + 4095) & ~4095); -#if !(defined(_MSC_VER) || defined(__MINGW32__)) - mprotect(hp_jitfunc, 4096, PROT_READ | PROT_WRITE | PROT_EXEC); -#endif } /** @@ -893,6 +895,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int if(hp_state == NULL) slow_hash_allocate_state(); + // locals to avoid constant TLS dereferencing + uint8_t *local_hp_state = hp_state; + v4_random_math_JIT_func local_hp_jitfunc = hp_jitfunc; + /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { memcpy(&state.hs, data, length); @@ -915,7 +921,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } else @@ -927,7 +933,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(j = 0; j < INIT_SIZE_BLK; j++) aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } @@ -975,7 +981,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } } else @@ -985,7 +991,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { for(j = 0; j < INIT_SIZE_BLK; j++) { - xor_blocks(&text[j * AES_BLOCK_SIZE], &hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + xor_blocks(&text[j * AES_BLOCK_SIZE], &local_hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); } } @@ -1065,24 +1071,24 @@ union cn_slow_hash_state #define pre_aes() \ j = state_index(a); \ - _c = vld1q_u8(&hp_state[j]); \ + _c = vld1q_u8(&local_hp_state[j]); \ _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ - VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ - VARIANT1_1(&hp_state[j]); \ + vst1q_u8(&local_hp_state[j], veorq_u8(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ - VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ @@ -1245,9 +1251,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int RDATA_ALIGN16 uint8_t expandedKey[240]; #ifndef FORCE_USE_HEAP - RDATA_ALIGN16 uint8_t hp_state[MEMORY]; + RDATA_ALIGN16 uint8_t local_hp_state[MEMORY]; #else - uint8_t *hp_state = (uint8_t *)aligned_malloc(MEMORY,16); + uint8_t *local_hp_state = (uint8_t *)aligned_malloc(MEMORY,16); #endif uint8_t text[INIT_SIZE_BYTE]; @@ -1287,7 +1293,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0]; @@ -1322,7 +1328,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } /* CryptoNight Step 5: Apply Keccak to the state again, and then @@ -1337,7 +1343,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int extra_hashes[state.hs.b[0] & 3](&state, 200, hash); #ifdef FORCE_USE_HEAP - aligned_free(hp_state); + aligned_free(local_hp_state); #endif } #else /* aarch64 && crypto */ diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index f40464bd1..094057b1f 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -282,6 +282,11 @@ namespace cryptonote //--------------------------------------------------------------- bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { + if (hwdev.compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki)) + { + return true; + } + if (ack.m_spend_secret_key == crypto::null_skey) { // for watch-only wallet, simply copy the known output pubkey @@ -982,15 +987,11 @@ namespace cryptonote { if (t.version == 1) return false; - static const crypto::hash empty_hash = { (char)0x70, (char)0xa4, (char)0x85, (char)0x5d, (char)0x04, (char)0xd8, (char)0xfa, (char)0x7b, (char)0x3b, (char)0x27, (char)0x82, (char)0xca, (char)0x53, (char)0xb6, (char)0x00, (char)0xe5, (char)0xc0, (char)0x03, (char)0xc7, (char)0xdc, (char)0xb2, (char)0x7d, (char)0x7e, (char)0x92, (char)0x3c, (char)0x23, (char)0xf7, (char)0x86, (char)0x01, (char)0x46, (char)0xd2, (char)0xc5 }; const unsigned int unprunable_size = t.unprunable_size; if (blob && unprunable_size) { CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); - if (blob->size() - unprunable_size == 0) - res = empty_hash; - else - cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); } else { @@ -1002,10 +1003,7 @@ namespace cryptonote const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - if (ss.str().empty()) - res = empty_hash; - else - cryptonote::get_blob_hash(ss.str(), res); + cryptonote::get_blob_hash(ss.str(), res); } return true; } @@ -1042,7 +1040,10 @@ namespace cryptonote } // prunable rct - hashes[2] = pruned_data_hash; + if (t.rct_signatures.type == rct::RCTTypeNull) + hashes[2] = crypto::null_hash; + else + hashes[2] = pruned_data_hash; // the tx hash is the hash of the 3 hashes crypto::hash res = cn_fast_hash(hashes, sizeof(hashes)); diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index ebfe45f37..89bca2f09 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -317,9 +317,12 @@ void HardFork::on_block_popped(uint64_t nblocks) uint64_t height; for (height = old_chain_height - 1; height >= new_chain_height; --height) { + version = versions.back(); + last_versions[version]--; versions.pop_back(); version = db.get_hard_fork_version(height); versions.push_front(version); + last_versions[version]++; } // does not take voting into account diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 3e903cd12..f5bd9bbb5 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1269,15 +1269,17 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_ if(h == 0) return; - m_db->block_txn_start(true); // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); - weights.reserve(weights.size() + h - start_offset); - for(size_t i = start_offset; i < h; i++) - { - weights.push_back(m_db->get_block_weight(i)); - } - m_db->block_txn_stop(); + weights = m_db->get_block_weights(start_offset, count); +} +//------------------------------------------------------------------ +void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + weights = m_db->get_long_term_block_weights(start_height, count); } //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_block_weight_limit() const @@ -3804,9 +3806,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons return block_weight; std::vector<uint64_t> weights; - weights.resize(nblocks); - for (uint64_t h = 0; h < nblocks; ++h) - weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h); + get_long_term_block_weights(weights, db_height - nblocks, nblocks); uint64_t long_term_median = epee::misc_utils::median(weights); uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); @@ -3850,9 +3850,7 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); if (nblocks == db_height) --nblocks; - weights.resize(nblocks); - for (uint64_t h = 0; h < nblocks; ++h) - weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1); + get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks); new_weights = weights; long_term_median = epee::misc_utils::median(weights); } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 4744defc5..3b8169764 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -1288,14 +1288,25 @@ namespace cryptonote /** * @brief gets recent block weights for median calculation * - * get the block weights of the last <count> blocks, and return by reference <sz>. + * get the block weights of the last <count> blocks, and return by reference <weights>. * - * @param sz return-by-reference the list of weights + * @param weights return-by-reference the list of weights * @param count the number of blocks to get weights for */ void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const; /** + * @brief gets recent block long term weights for median calculation + * + * get the block long term weights of the last <count> blocks, and return by reference <weights>. + * + * @param weights return-by-reference the list of weights + * @param start_height the block height of the first block to query + * @param count the number of blocks to get weights for + */ + void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const; + + /** * @brief checks if a transaction is unlocked (its outputs spendable) * * This function checks to see if a transaction is unlocked. diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 58acdb6bb..387203cc0 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -921,6 +921,7 @@ namespace cryptonote catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_pre: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); @@ -951,6 +952,7 @@ namespace cryptonote catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_post: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); diff --git a/src/device/device.hpp b/src/device/device.hpp index 7f1a60c9f..65b38361b 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -70,6 +70,7 @@ namespace cryptonote struct account_keys; struct subaddress_index; struct tx_destination_entry; + struct keypair; } namespace hw { @@ -81,11 +82,18 @@ namespace hw { return false; } + class device_progress { + public: + virtual double progress() const { return 0; } + virtual bool indeterminate() const { return false; } + }; + class i_device_callback { public: - virtual void on_button_request() {} - virtual void on_pin_request(epee::wipeable_string & pin) {} - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual void on_button_request(uint64_t code=0) {} + virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; } + virtual void on_progress(const device_progress& event) {} virtual ~i_device_callback() = default; }; @@ -141,6 +149,9 @@ namespace hw { virtual void set_callback(i_device_callback * callback) {}; virtual void set_derivation_path(const std::string &derivation_path) {}; + virtual void set_pin(const epee::wipeable_string & pin) {} + virtual void set_passphrase(const epee::wipeable_string & passphrase) {} + /* ======================================================================= */ /* LOCKER */ /* ======================================================================= */ @@ -229,7 +240,9 @@ namespace hw { virtual bool has_ki_cold_sync(void) const { return false; } virtual bool has_tx_cold_sign(void) const { return false; } - + virtual bool has_ki_live_refresh(void) const { return true; } + virtual bool compute_key_image(const cryptonote::account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const cryptonote::subaddress_index& received_index, cryptonote::keypair& in_ephemeral, crypto::key_image& ki) { return false; } + virtual void computing_key_images(bool started) {}; virtual void set_network_type(cryptonote::network_type network_type) { } protected: diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index 8b8cdf6d2..31b1504ab 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -31,6 +31,7 @@ #define MONERO_DEVICE_COLD_H #include "wallet/wallet2.h" +#include <boost/optional/optional.hpp> #include <boost/function.hpp> @@ -44,6 +45,8 @@ namespace hw { public: std::vector<std::string> tx_device_aux; // device generated aux data std::vector<cryptonote::address_parse_info> tx_recipients; // as entered by user + boost::optional<int> bp_version; // BP version to use + boost::optional<unsigned> client_version; // Signing client version to use (testing) }; class device_cold { @@ -51,6 +54,53 @@ namespace hw { using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>; + class op_progress : public hw::device_progress { + public: + op_progress():m_progress(0), m_indeterminate(false) {}; + explicit op_progress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate){} + + double progress() const override { return m_progress; } + bool indeterminate() const override { return m_indeterminate; } + protected: + double m_progress; + bool m_indeterminate; + }; + + class tx_progress : public op_progress { + public: + tx_progress(): + m_cur_tx(0), m_max_tx(1), + m_cur_step(0), m_max_step(1), + m_cur_substep(0), m_max_substep(1){}; + + tx_progress(size_t cur_tx, size_t max_tx, size_t cur_step, size_t max_step, size_t cur_substep, size_t max_substep): + m_cur_tx(cur_tx), m_max_tx(max_tx), + m_cur_step(cur_tx), m_max_step(max_tx), + m_cur_substep(cur_tx), m_max_substep(max_tx){} + + double progress() const override { + return std::max(1.0, (double)m_cur_tx / m_max_tx + + (double)m_cur_step / (m_max_tx * m_max_step) + + (double)m_cur_substep / (m_max_tx * m_max_step * m_max_substep)); + } + bool indeterminate() const override { return false; } + + protected: + size_t m_cur_tx; + size_t m_max_tx; + size_t m_cur_step; + size_t m_max_step; + size_t m_cur_substep; + size_t m_max_substep; + }; + + typedef struct { + std::string salt1; + std::string salt2; + std::string tx_enc_keys; + std::string tx_prefix_hash; + } tx_key_data_t; + /** * Key image sync with the cold protocol. */ @@ -65,6 +115,52 @@ namespace hw { const ::tools::wallet2::unsigned_tx_set & unsigned_tx, ::tools::wallet2::signed_tx_set & signed_tx, tx_aux_data & aux_data) =0; + + /** + * Get tx key support check. + */ + virtual bool is_get_tx_key_supported() const { return false; } + + /** + * Loads TX aux data required for tx key. + */ + virtual void load_tx_key_data(tx_key_data_t & res, const std::string & tx_aux_data) =0; + + /** + * Decrypts TX keys. + */ + virtual void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) =0; + + /** + * Live refresh support check + */ + virtual bool is_live_refresh_supported() const { return false; }; + + /** + * Starts live refresh process with the device + */ + virtual void live_refresh_start() =0; + + /** + * One live refresh step + */ + virtual void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) =0; + + /** + * Live refresh process termination + */ + virtual void live_refresh_finish() =0; }; } diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index ceb6111e0..b4a80cf2c 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -57,7 +57,9 @@ namespace trezor { } device_trezor::device_trezor() { - + m_live_refresh_in_progress = false; + m_live_refresh_enabled = true; + m_live_refresh_thread_running = false; } device_trezor::~device_trezor() { @@ -69,6 +71,89 @@ namespace trezor { } } + bool device_trezor::init() + { + m_live_refresh_in_progress = false; + bool r = device_trezor_base::init(); + if (r && !m_live_refresh_thread) + { + m_live_refresh_thread_running = true; + m_live_refresh_thread.reset(new boost::thread(boost::bind(&device_trezor::live_refresh_thread_main, this))); + } + return r; + } + + bool device_trezor::release() + { + m_live_refresh_in_progress = false; + m_live_refresh_thread_running = false; + if (m_live_refresh_thread) + { + m_live_refresh_thread->join(); + m_live_refresh_thread = nullptr; + } + return device_trezor_base::release(); + } + + bool device_trezor::disconnect() + { + m_live_refresh_in_progress = false; + return device_trezor_base::disconnect(); + } + + void device_trezor::device_state_reset_unsafe() + { + require_connected(); + if (m_live_refresh_in_progress) + { + try + { + live_refresh_finish_unsafe(); + } + catch(const std::exception & e) + { + MERROR("Live refresh could not be terminated: " << e.what()); + } + } + + m_live_refresh_in_progress = false; + device_trezor_base::device_state_reset_unsafe(); + } + + void device_trezor::live_refresh_thread_main() + { + while(m_live_refresh_thread_running) + { + boost::this_thread::sleep_for(boost::chrono::milliseconds(100)); + if (!m_live_refresh_in_progress) + { + continue; + } + + TREZOR_AUTO_LOCK_DEVICE(); + if (!m_transport || !m_live_refresh_in_progress) + { + continue; + } + + auto current_time = std::chrono::steady_clock::now(); + if (current_time - m_last_live_refresh_time <= std::chrono::seconds(20)) + { + continue; + } + + MTRACE("Closing live refresh process due to inactivity"); + try + { + live_refresh_finish(); + } + catch(const std::exception &e) + { + MWARNING("Live refresh auto-finish failed: " << e.what()); + } + } + } + /* ======================================================================= */ /* WALLET & ADDRESS */ /* ======================================================================= */ @@ -126,7 +211,7 @@ namespace trezor { std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address( const boost::optional<std::vector<uint32_t>> & path, const boost::optional<cryptonote::network_type> & network_type){ - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -142,7 +227,7 @@ namespace trezor { std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key( const boost::optional<std::vector<uint32_t>> & path, const boost::optional<cryptonote::network_type> & network_type){ - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -155,11 +240,43 @@ namespace trezor { return response; } + bool device_trezor::is_get_tx_key_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); + } + + void device_trezor::load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) + { + protocol::tx::load_tx_key_data(res, tx_aux_data); + } + + void device_trezor::get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = protocol::tx::get_tx_key(tx_aux_data); + this->set_msg_addr<messages::monero::MoneroGetTxKeyRequest>(req.get()); + + auto response = this->client_exchange<messages::monero::MoneroGetTxKeyAck>(req); + MTRACE("Get TX key response received"); + + protocol::tx::get_tx_key_ack(tx_keys, tx_aux_data.tx_prefix_hash, view_key_priv, response); + } + void device_trezor::ki_sync(wallet_shim * wallet, const std::vector<tools::wallet2::transfer_details> & transfers, hw::device_cold::exported_key_image & ski) { - AUTO_LOCK_CMD(); +#define EVENT_PROGRESS(P) do { if (m_callback) {(m_callback)->on_progress(device_cold::op_progress(P)); } }while(0) + + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -171,6 +288,7 @@ namespace trezor { protocol::ki::key_image_data(wallet, transfers, mtds); protocol::ki::generate_commitment(mtds, transfers, req); + EVENT_PROGRESS(0.); this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req); @@ -194,27 +312,160 @@ namespace trezor { } MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + EVENT_PROGRESS((double)cur * batch_size / mtds.size()); } + EVENT_PROGRESS(1.); auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>(); auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req); ski.reserve(kis.size()); for(auto & sub : kis){ - char buff[32*3]; + ::crypto::signature sig{}; + ::crypto::key_image ki; + char buff[sizeof(ki.data)*3]; + + size_t buff_len = sizeof(buff); + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()), - reinterpret_cast<const uint8_t *>(sub.iv().data()), buff); + reinterpret_cast<const uint8_t *>(sub.iv().data()), buff, &buff_len); + CHECK_AND_ASSERT_THROW_MES(buff_len == sizeof(buff), "Plaintext size invalid"); - ::crypto::signature sig{}; - ::crypto::key_image ki; - memcpy(ki.data, buff, 32); - memcpy(sig.c.data, buff + 32, 32); - memcpy(sig.r.data, buff + 64, 32); + memcpy(ki.data, buff, sizeof(ki.data)); + memcpy(sig.c.data, buff + sizeof(ki.data), sizeof(ki.data)); + memcpy(sig.r.data, buff + 2*sizeof(ki.data), sizeof(ki.data)); ski.push_back(std::make_pair(ki, sig)); } +#undef EVENT_PROGRESS + } + + bool device_trezor::is_live_refresh_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); } + bool device_trezor::is_live_refresh_enabled() const + { + return is_live_refresh_supported() && (mode == NONE || mode == TRANSACTION_PARSE) && m_live_refresh_enabled; + } + + bool device_trezor::has_ki_live_refresh() const + { + try{ + return is_live_refresh_enabled(); + } catch(const std::exception & e){ + MERROR("Could not detect if live refresh is enabled: " << e.what()); + } + return false; + } + + void device_trezor::live_refresh_start() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + live_refresh_start_unsafe(); + } + + void device_trezor::live_refresh_start_unsafe() + { + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>(); + this->set_msg_addr<messages::monero::MoneroLiveRefreshStartRequest>(req.get()); + this->client_exchange<messages::monero::MoneroLiveRefreshStartAck>(req); + m_live_refresh_in_progress = true; + m_last_live_refresh_time = std::chrono::steady_clock::now(); + } + + void device_trezor::live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + + if (!m_live_refresh_in_progress) + { + live_refresh_start_unsafe(); + } + + m_last_live_refresh_time = std::chrono::steady_clock::now(); + + auto req = std::make_shared<messages::monero::MoneroLiveRefreshStepRequest>(); + req->set_out_key(out_key.data, 32); + req->set_recv_deriv(recv_derivation.data, 32); + req->set_real_out_idx(real_output_index); + req->set_sub_addr_major(received_index.major); + req->set_sub_addr_minor(received_index.minor); + + auto ack = this->client_exchange<messages::monero::MoneroLiveRefreshStepAck>(req); + protocol::ki::live_refresh_ack(view_key_priv, out_key, ack, in_ephemeral, ki); + } + + void device_trezor::live_refresh_finish_unsafe() + { + auto req = std::make_shared<messages::monero::MoneroLiveRefreshFinalRequest>(); + this->client_exchange<messages::monero::MoneroLiveRefreshFinalAck>(req); + m_live_refresh_in_progress = false; + } + + void device_trezor::live_refresh_finish() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + if (m_live_refresh_in_progress) + { + live_refresh_finish_unsafe(); + } + } + + void device_trezor::computing_key_images(bool started) + { + try + { + if (!is_live_refresh_enabled()) + { + return; + } + + // React only on termination as the process can auto-start itself. + if (!started && m_live_refresh_in_progress) + { + live_refresh_finish(); + } + } + catch(const std::exception & e) + { + MWARNING("KI computation state change failed, started: " << started << ", e: " << e.what()); + } + } + + bool device_trezor::compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + if (!is_live_refresh_enabled()) + { + return false; + } + + live_refresh(ack.m_view_secret_key, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki); + return true; + } void device_trezor::tx_sign(wallet_shim * wallet, const tools::wallet2::unsigned_tx_set & unsigned_tx, @@ -222,7 +473,15 @@ namespace trezor { hw::tx_aux_data & aux_data) { CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); - size_t num_tx = unsigned_tx.txes.size(); + + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + transaction_versions_check(unsigned_tx, aux_data); + + const size_t num_tx = unsigned_tx.txes.size(); + m_num_transations_to_sign = num_tx; signed_tx.key_images.clear(); signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); @@ -267,6 +526,10 @@ namespace trezor { cpend.key_images = key_images; // KI sync + for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){ + signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image; + } + size_t num_sources = cdata.tx_data.sources.size(); CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); @@ -276,12 +539,19 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; - auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); + CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset"); + idx_map_src -= unsigned_tx.transfers.first; CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + + const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); signed_tx.key_images[idx_map_src] = vini.k_image; } } + + if (m_callback){ + m_callback->on_progress(device_cold::tx_progress(m_num_transations_to_sign, m_num_transations_to_sign, 1, 1, 1, 1)); + } } void device_trezor::tx_sign(wallet_shim * wallet, @@ -290,10 +560,16 @@ namespace trezor { hw::tx_aux_data & aux_data, std::shared_ptr<protocol::tx::Signer> & signer) { - AUTO_LOCK_CMD(); +#define EVENT_PROGRESS(S, SUB, SUBMAX) do { if (m_callback) { \ + (m_callback)->on_progress(device_cold::tx_progress(idx, m_num_transations_to_sign, S, 10, SUB, SUBMAX)); \ +} }while(0) + require_connected(); - device_state_reset_unsafe(); + if (idx > 0) + device_state_reset_unsafe(); + require_initialized(); + EVENT_PROGRESS(0, 1, 1); CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data); @@ -305,6 +581,7 @@ namespace trezor { auto init_msg = signer->step_init(); this->set_msg_addr(init_msg.get()); transaction_pre_check(init_msg); + EVENT_PROGRESS(1, 1, 1); auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg); signer->step_init_ack(response); @@ -314,6 +591,7 @@ namespace trezor { auto src = signer->step_set_input(cur_src); auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src); signer->step_set_input_ack(ack); + EVENT_PROGRESS(2, cur_src, num_sources); } // Step: sort @@ -322,44 +600,82 @@ namespace trezor { auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req); signer->step_permutation_ack(perm_ack); } + EVENT_PROGRESS(3, 1, 1); // Step: input_vini - if (!signer->in_memory()){ - for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ - auto src = signer->step_set_vini_input(cur_src); - auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); - signer->step_set_vini_input_ack(ack); - } + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); + signer->step_set_vini_input_ack(ack); + EVENT_PROGRESS(4, cur_src, num_sources); } // Step: all inputs set auto all_inputs_set = signer->step_all_inputs_set(); auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set); signer->step_all_inputs_set_ack(ack_all_inputs); + EVENT_PROGRESS(5, 1, 1); // Step: outputs for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ auto src = signer->step_set_output(cur_dst); auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src); signer->step_set_output_ack(ack); + + // If BP is offloaded to host, another step with computed BP may be needed. + auto offloaded_bp = signer->step_rsig(cur_dst); + if (offloaded_bp){ + auto bp_ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(offloaded_bp); + signer->step_set_rsig_ack(ack); + } + + EVENT_PROGRESS(6, cur_dst, num_outputs); } // Step: all outs set auto all_out_set = signer->step_all_outs_set(); auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set); signer->step_all_outs_set_ack(ack_all_out_set, *this); + EVENT_PROGRESS(7, 1, 1); // Step: sign each input for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ auto src = signer->step_sign_input(cur_src); auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src); signer->step_sign_input_ack(ack_sign); + EVENT_PROGRESS(8, cur_src, num_sources); } // Step: final auto final_msg = signer->step_final(); auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg); signer->step_final_ack(ack_final); + EVENT_PROGRESS(9, 1, 1); +#undef EVENT_PROGRESS + } + + void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data) + { + auto trezor_version = get_version(); + unsigned client_version = 1; // default client version for tx + + if (trezor_version <= pack_version(2, 0, 10)){ + client_version = 0; + } + + if (aux_data.client_version){ + auto wanted_client_version = aux_data.client_version.get(); + if (wanted_client_version > client_version){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update."); + } else { + client_version = wanted_client_version; + } + } + aux_data.client_version = client_version; + + if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update."); + } } void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg) diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 75f6d5875..0e91847dc 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -30,18 +30,21 @@ #ifndef MONERO_DEVICE_TREZOR_H #define MONERO_DEVICE_TREZOR_H +#include "trezor.hpp" +#include "device/device.hpp" +#ifdef WITH_DEVICE_TREZOR #include <cstddef> #include <string> -#include "device/device.hpp" -#include "device/device_default.hpp" -#include "device/device_cold.hpp" #include <boost/scope_exit.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> + +#include "device/device_default.hpp" +#include "device/device_cold.hpp" #include "cryptonote_config.h" -#include "trezor.hpp" #include "device_trezor_base.hpp" +#endif namespace hw { namespace trezor { @@ -57,8 +60,29 @@ namespace trezor { */ class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { protected: + std::atomic<bool> m_live_refresh_in_progress; + std::chrono::steady_clock::time_point m_last_live_refresh_time; + std::unique_ptr<boost::thread> m_live_refresh_thread; + std::atomic<bool> m_live_refresh_thread_running; + bool m_live_refresh_enabled; + size_t m_num_transations_to_sign; + + void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data); void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg); void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); + void device_state_reset_unsafe() override; + void live_refresh_start_unsafe(); + void live_refresh_finish_unsafe(); + void live_refresh_thread_main(); + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer); public: device_trezor(); @@ -69,11 +93,17 @@ namespace trezor { explicit operator bool() const override {return true;} + bool init() override; + bool release() override; + bool disconnect() override; + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; bool has_ki_cold_sync() const override { return true; } bool has_tx_cold_sign() const override { return true; } void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + void set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; } + bool live_refresh_enabled() const { return m_live_refresh_enabled; } /* ======================================================================= */ /* WALLET & ADDRESS */ @@ -100,20 +130,68 @@ namespace trezor { const boost::optional<cryptonote::network_type> & network_type = boost::none); /** + * Get_tx_key support check + */ + bool is_get_tx_key_supported() const override; + + /** + * Loads tx aux data + */ + void load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) override; + + /** + * TX key load with the Trezor + */ + void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) override; + + /** * Key image sync with the Trezor. */ void ki_sync(wallet_shim * wallet, const std::vector<::tools::wallet2::transfer_details> & transfers, hw::device_cold::exported_key_image & ski) override; + bool is_live_refresh_supported() const override; + + bool is_live_refresh_enabled() const; + + bool has_ki_live_refresh() const override; + + void live_refresh_start() override; + + void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) override; + + void live_refresh_finish() override; + /** - * Signs particular transaction idx in the unsigned set, keeps state in the signer + * Letting device know the KI computation started / ended. + * During refresh */ - void tx_sign(wallet_shim * wallet, - const ::tools::wallet2::unsigned_tx_set & unsigned_tx, - size_t idx, - hw::tx_aux_data & aux_data, - std::shared_ptr<protocol::tx::Signer> & signer); + void computing_key_images(bool started) override; + + /** + * Implements hw::device interface + * called from generate_key_image_helper_precomp() + */ + bool compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) override; /** * Signs unsigned transaction with the Trezor. diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 1892b47d9..f3d15c5e2 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -109,6 +109,7 @@ namespace trezor { disconnect(); // Enumerate all available devices + TREZOR_AUTO_LOCK_DEVICE(); try { hw::trezor::t_transport_vect trans; @@ -145,6 +146,7 @@ namespace trezor { } bool device_trezor_base::disconnect() { + TREZOR_AUTO_LOCK_DEVICE(); m_device_state.clear(); m_features.reset(); @@ -203,13 +205,13 @@ namespace trezor { /* Helpers */ /* ======================================================================= */ - void device_trezor_base::require_connected(){ + void device_trezor_base::require_connected() const { if (!m_transport){ throw exc::NotConnectedException(); } } - void device_trezor_base::require_initialized(){ + void device_trezor_base::require_initialized() const { if (!m_features){ throw exc::TrezorException("Device state not initialized"); } @@ -330,7 +332,7 @@ namespace trezor { /* ======================================================================= */ bool device_trezor_base::ping() { - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); if (!m_transport){ MINFO("Ping failed, device not connected"); return false; @@ -364,7 +366,7 @@ namespace trezor { void device_trezor_base::device_state_reset() { - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); device_state_reset_unsafe(); } @@ -373,6 +375,10 @@ namespace trezor { if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \ if (m_callback) m_callback->method(__VA_ARGS__); \ }while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) do { \ + if (m_debug_callback) VAR = m_debug_callback->method(__VA_ARGS__); \ + if (m_callback) VAR = m_callback->method(__VA_ARGS__); \ +}while(0) void device_trezor_base::setup_debug(){ if (!m_debug){ @@ -392,6 +398,7 @@ namespace trezor { #else #define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) VAR = (m_callback ? m_callback->method(__VA_ARGS__) : boost::none) #endif void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) @@ -402,7 +409,7 @@ namespace trezor { messages::common::ButtonAck ack; write_raw(&ack); - TREZOR_CALLBACK(on_button_request); + TREZOR_CALLBACK(on_button_request, msg->code()); resp = read_raw(); } @@ -411,13 +418,18 @@ namespace trezor { MDEBUG("on_pin_request"); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - epee::wipeable_string pin; + boost::optional<epee::wipeable_string> pin; + TREZOR_CALLBACK_GET(pin, on_pin_request); - TREZOR_CALLBACK(on_pin_request, pin); + if (!pin && m_pin){ + pin = m_pin; + } // TODO: remove PIN from memory messages::common::PinMatrixAck m; - m.set_pin(pin.data(), pin.size()); + if (pin) { + m.set_pin(pin.get().data(), pin.get().size()); + } resp = call_raw(&m); } @@ -425,14 +437,19 @@ namespace trezor { { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); MDEBUG("on_passhprase_request, on device: " << msg->on_device()); - epee::wipeable_string passphrase; + boost::optional<epee::wipeable_string> passphrase; + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device()); - TREZOR_CALLBACK(on_passphrase_request, msg->on_device(), passphrase); + if (!passphrase && m_passphrase){ + passphrase = m_passphrase; + } + + m_passphrase = boost::none; messages::common::PassphraseAck m; - if (!msg->on_device()){ + if (!msg->on_device() && passphrase){ // TODO: remove passphrase from memory - m.set_passphrase(passphrase.data(), passphrase.size()); + m.set_passphrase(passphrase.get().data(), passphrase.get().size()); } if (!m_device_state.empty()){ @@ -494,16 +511,16 @@ namespace trezor { m_debug_link->init(debug_transport); } - void trezor_debug_callback::on_button_request() { + void trezor_debug_callback::on_button_request(uint64_t code) { if (m_debug_link) m_debug_link->press_yes(); } - void trezor_debug_callback::on_pin_request(epee::wipeable_string &pin) { - + boost::optional<epee::wipeable_string> trezor_debug_callback::on_pin_request() { + return boost::none; } - void trezor_debug_callback::on_passphrase_request(bool on_device, epee::wipeable_string &passphrase) { - + boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) { + return boost::none; } void trezor_debug_callback::on_passphrase_state_request(const std::string &state) { diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 8942df797..8c3c14b29 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -47,14 +47,15 @@ #endif //automatic lock one more level on device ensuring the current thread is allowed to use it -#define AUTO_LOCK_CMD() \ +#define TREZOR_AUTO_LOCK_CMD() \ /* lock both mutexes without deadlock*/ \ boost::lock(device_locker, command_locker); \ /* make sure both already-locked mutexes are unlocked at the end of scope */ \ boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) - +#define TREZOR_AUTO_LOCK_DEVICE() boost::lock_guard<boost::recursive_mutex> lock1_device(device_locker) + namespace hw { namespace trezor { @@ -62,14 +63,14 @@ namespace trezor { class device_trezor_base; #ifdef WITH_TREZOR_DEBUGGING - class trezor_debug_callback { + class trezor_debug_callback : public hw::i_device_callback { public: trezor_debug_callback()=default; explicit trezor_debug_callback(std::shared_ptr<Transport> & debug_transport); - void on_button_request(); - void on_pin_request(epee::wipeable_string &pin); - void on_passphrase_request(bool on_device, epee::wipeable_string &passphrase); + void on_button_request(uint64_t code=0) override; + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; void on_passphrase_state_request(const std::string &state); void on_disconnect(); protected: @@ -95,6 +96,8 @@ namespace trezor { std::vector<unsigned int> m_wallet_deriv_path; std::string m_device_state; // returned after passphrase entry, session std::shared_ptr<messages::management::Features> m_features; // features from the last device reset + boost::optional<epee::wipeable_string> m_pin; + boost::optional<epee::wipeable_string> m_passphrase; cryptonote::network_type network_type; @@ -109,11 +112,11 @@ namespace trezor { // Internal methods // - void require_connected(); - void require_initialized(); + void require_connected() const; + void require_initialized() const; void call_ping_unsafe(); void test_ping(); - void device_state_reset_unsafe(); + virtual void device_state_reset_unsafe(); void ensure_derivation_path() noexcept; // Communication methods @@ -265,6 +268,15 @@ namespace trezor { void set_derivation_path(const std::string &deriv_path) override; + virtual bool has_ki_live_refresh(void) const override { return false; } + + virtual void set_pin(const epee::wipeable_string & pin) override { + m_pin = pin; + } + virtual void set_passphrase(const epee::wipeable_string & passphrase) override { + m_passphrase = passphrase; + } + /* ======================================================================= */ /* SETUP/TEARDOWN */ /* ======================================================================= */ diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 79f6adac9..db2444df6 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -33,6 +33,8 @@ #include <utility> #include <boost/endian/conversion.hpp> #include <common/apply_permutation.h> +#include <common/json_util.h> +#include <crypto/hmac-keccak.h> #include <ringct/rctSigs.h> #include <ringct/bulletproofs.h> #include "cryptonote_config.h" @@ -40,6 +42,37 @@ #include <sodium/crypto_verify_32.h> #include <sodium/crypto_aead_chacha20poly1305.h> +#define GET_FIELD_STRING(name, type, jtype) field_##name = std::string(json[#name].GetString(), json[#name].GetStringLength()) +#define GET_FIELD_OTHER(name, type, jtype) field_##name = static_cast<type>(json[#name].Get##jtype()) + +#define GET_STRING_FROM_JSON(json, name, type, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, String, mandatory, def, GET_FIELD_STRING) + +#define GET_FIELD_FROM_JSON(json, name, type, jtype, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, GET_FIELD_OTHER) + +#define GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, VAL) \ + type field_##name = static_cast<type>(def); \ + bool field_##name##_found = false; \ + (void)field_##name##_found; \ + do if (json.HasMember(#name)) \ + { \ + if (json[#name].Is##jtype()) \ + { \ + VAL(name, type, jtype); \ + field_##name##_found = true; \ + } \ + else \ + { \ + throw std::invalid_argument("Field " #name " found in JSON, but not " #jtype); \ + } \ + } \ + else if (mandatory) \ + { \ + throw std::invalid_argument("Field " #name " not found in JSON");\ + } while(0) + + namespace hw{ namespace trezor{ namespace protocol{ @@ -84,19 +117,22 @@ namespace protocol{ namespace crypto { namespace chacha { - void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){ - if (length < 16){ - throw std::invalid_argument("Ciphertext length too small"); - } + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len){ + CHECK_AND_ASSERT_THROW_MES(length >= TAG_SIZE, "Ciphertext length too small"); + CHECK_AND_ASSERT_THROW_MES(!plaintext_len || *plaintext_len >= (length - TAG_SIZE), "Plaintext length too small"); - unsigned long long int cip_len = length; + unsigned long long int res_len = plaintext_len ? *plaintext_len : length; auto r = crypto_aead_chacha20poly1305_ietf_decrypt( - reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr, + reinterpret_cast<unsigned char *>(plaintext), &res_len, nullptr, static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key); if (r != 0){ throw exc::Poly1305TagInvalid(); } + + if (plaintext_len){ + *plaintext_len = (size_t) res_len; + } } } @@ -185,6 +221,49 @@ namespace ki { } } + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + std::string str_out_key(out_key.data, sizeof(out_key.data)); + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, str_out_key, ack->salt()); + + const size_t len_ciphertext = ack->key_image().size(); // IV || keys + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t ki_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[ki_len]); + uint8_t * buff = plaintext.get(); + + protocol::crypto::chacha::decrypt( + ack->key_image().data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast<const uint8_t *>(enc_key.data), + reinterpret_cast<const uint8_t *>(ack->key_image().data()), + reinterpret_cast<char *>(buff), &ki_len); + + CHECK_AND_ASSERT_THROW_MES(ki_len == 3*32, "Invalid size"); + ::crypto::signature sig{}; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + in_ephemeral.pub = out_key; + in_ephemeral.sec = ::crypto::null_skey; + + // Verification + std::vector<const ::crypto::public_key*> pkeys; + pkeys.push_back(&out_key); + + CHECK_AND_ASSERT_THROW_MES(rct::scalarmultKey(rct::ki2rct(ki), rct::curveOrder()) == rct::identity(), + "Key image out of validity domain: key image " << epee::string_tools::pod_to_hex(ki)); + + CHECK_AND_ASSERT_THROW_MES(::crypto::check_ring_signature((const ::crypto::hash&)ki, ki, pkeys, &sig), + "Signature failed for key image " << epee::string_tools::pod_to_hex(ki) + << ", signature " + epee::string_tools::pod_to_hex(sig) + << ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } } // Cold transaction signing @@ -198,6 +277,8 @@ namespace tx { void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ dst->set_amount(src->amount); dst->set_is_subaddress(src->is_subaddress); + dst->set_is_integrated(src->is_integrated); + dst->set_original(src->original); translate_address(dst->mutable_addr(), &(src->addr)); } @@ -267,9 +348,29 @@ namespace tx { return std::string(buff, offset); } + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt) + { + uint8_t hash[32]; + KECCAK_CTX ctx; + ::crypto::secret_key res; + + keccak_init(&ctx); + keccak_update(&ctx, (const uint8_t *) private_view_key.data, sizeof(private_view_key.data)); + if (!aux.empty()){ + keccak_update(&ctx, (const uint8_t *) aux.data(), aux.size()); + } + keccak_finish(&ctx, hash); + keccak(hash, sizeof(hash), hash, sizeof(hash)); + + hmac_keccak_hash(hash, (const uint8_t *) salt.data(), salt.size(), hash, sizeof(hash)); + memcpy(res.data, hash, sizeof(hash)); + memwipe(hash, sizeof(hash)); + return res; + } + TData::TData() { - in_memory = false; rsig_type = 0; + bp_version = 0; cur_input_idx = 0; cur_output_idx = 0; cur_batch_idx = 0; @@ -283,6 +384,7 @@ namespace tx { m_tx_idx = tx_idx; m_ct.tx_data = cur_tx(); m_multisig = false; + m_client_version = 1; } void Signer::extract_payment_id(){ @@ -392,8 +494,10 @@ namespace tx { m_ct.tx.version = 2; m_ct.tx.unlock_time = tx.unlock_time; + m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1); tsx_data.set_version(1); + tsx_data.set_client_version(client_version()); tsx_data.set_unlock_time(tx.unlock_time); tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); @@ -404,6 +508,10 @@ namespace tx { auto rsig_data = tsx_data.mutable_rsig_data(); m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size()); rsig_data->set_rsig_type(m_ct.rsig_type); + if (tx.use_bulletproofs){ + m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1); + rsig_data->set_bp_version((uint32_t) m_ct.bp_version); + } generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); @@ -437,7 +545,6 @@ namespace tx { } void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){ - m_ct.in_memory = false; if (ack->has_rsig_data()){ m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data()); } @@ -505,10 +612,6 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ sort_ki(); - if (in_memory()){ - return nullptr; - } - auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); @@ -516,15 +619,10 @@ namespace tx { } void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){ - if (in_memory()){ - return; - } + } std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){ - if (in_memory()){ - return nullptr; - } CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); @@ -536,7 +634,8 @@ namespace tx { translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); - if (!in_memory()) { + + if (client_version() == 0) { CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); res->set_pseudo_out(m_ct.pseudo_outs[idx]); @@ -547,9 +646,7 @@ namespace tx { } void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){ - if (in_memory()){ - return; - } + } std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){ @@ -557,34 +654,37 @@ namespace tx { } void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ - if (is_offloading()){ - // If offloading, expect rsig configuration. - if (!ack->has_rsig_data()){ - throw exc::ProtocolException("Rsig offloading requires rsig param"); - } + if (client_version() > 0 || !is_offloading()){ + return; + } - auto & rsig_data = ack->rsig_data(); - if (!rsig_data.has_mask()){ - throw exc::ProtocolException("Gamma masks not present in offloaded version"); - } + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } - auto & mask = rsig_data.mask(); - if (mask.size() != 32 * num_outputs()){ - throw exc::ProtocolException("Invalid number of gamma masks"); - } + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } - m_ct.rsig_gamma.reserve(num_outputs()); - for(size_t c=0; c < num_outputs(); ++c){ - rct::key cmask{}; - memcpy(cmask.bytes, mask.data() + c * 32, 32); - m_ct.rsig_gamma.emplace_back(cmask); - } + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); } } std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported"); m_ct.cur_output_idx = idx; m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() @@ -595,48 +695,11 @@ namespace tx { res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); // Range sig offloading to the host - if (!is_offloading()) { - return res; - } - - CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); - if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) { - return res; - } - - auto rsig_data = res->mutable_rsig_data(); - auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; - - if (!is_req_bulletproof()){ - if (batch_size > 1){ - throw std::invalid_argument("Borromean cannot batch outputs"); - } - - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - rct::key C{}, mask = m_ct.rsig_gamma[idx]; - auto genRsig = rct::proveRange(C, mask, cur_dst.amount); // TODO: rsig with given mask - auto serRsig = cn_serialize(genRsig); - m_ct.tx_out_rsigs.emplace_back(genRsig); - rsig_data->set_rsig(serRsig); - - } else { - std::vector<uint64_t> amounts; - rct::keyV masks; - CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching"); - - for(size_t i = 0; i < batch_size; ++i){ - const size_t bidx = 1 + idx - batch_size + i; - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - - amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); - masks.push_back(m_ct.rsig_gamma[bidx]); - } - - auto bp = bulletproof_PROVE(amounts, masks); - auto serRsig = cn_serialize(bp); - m_ct.tx_out_rsigs.emplace_back(bp); - rsig_data->set_rsig(serRsig); + // ClientV0 sends offloaded BP with the last message in the batch. + // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks. + if (client_version() == 0 && is_offloading() && should_compute_bp_now()) { + auto rsig_data = res->mutable_rsig_data(); + compute_bproof(*rsig_data); } return res; @@ -644,7 +707,6 @@ namespace tx { void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ cryptonote::tx_out tx_out; - rct::rangeSig range_sig{}; rct::Bulletproof bproof{}; rct::ctkey out_pk{}; rct::ecdhTuple ecdh{}; @@ -658,12 +720,12 @@ namespace tx { if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ has_rsig = true; rsig_buff = rsig_data.rsig(); + } - } else if (rsig_data.rsig_parts_size() > 0){ - has_rsig = true; - for (const auto &it : rsig_data.rsig_parts()) { - rsig_buff += it; - } + if (client_version() >= 1 && rsig_data.has_mask()){ + rct::key cmask{}; + string_to_key(cmask, rsig_data.mask()); + m_ct.rsig_gamma.emplace_back(cmask); } } @@ -675,12 +737,13 @@ namespace tx { throw exc::ProtocolException("Cannot deserialize out_pk"); } - if (!cn_deserialize(ack->ecdh_info(), ecdh)){ - throw exc::ProtocolException("Cannot deserialize ecdhtuple"); - } - - if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){ - throw exc::ProtocolException("Cannot deserialize rangesig"); + if (m_ct.bp_version <= 1) { + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + } else { + CHECK_AND_ASSERT_THROW_MES(8 == ack->ecdh_info().size(), "Invalid ECDH.amount size"); + memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8); } if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ @@ -692,35 +755,77 @@ namespace tx { m_ct.tx_out_pk.emplace_back(out_pk); m_ct.tx_out_ecdh.emplace_back(ecdh); - if (!has_rsig){ + // ClientV0, if no rsig was generated on Trezor, do not continue. + // ClientV1+ generates BP after all masks in the current batch are generated + if (!has_rsig || (client_version() >= 1 && is_offloading())){ return; } - if (is_req_bulletproof()){ - CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); - auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; - for (size_t i = 0; i < batch_size; ++i){ - const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + process_bproof(bproof); + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } - rct::key commitment = m_ct.tx_out_pk[bidx].mask; - commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); - bproof.V.push_back(commitment); - } + bool Signer::should_compute_bp_now() const { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + return m_ct.grouping_vct[m_ct.cur_batch_idx] <= m_ct.cur_output_in_batch_idx; + } - m_ct.tx_out_rsigs.emplace_back(bproof); - if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); - } + void Signer::compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data){ + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + std::vector<uint64_t> amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_output_idx + 1 >= batch_size, "Invalid index for batching"); - } else { - m_ct.tx_out_rsigs.emplace_back(range_sig); + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); - } + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); } + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data.set_rsig(serRsig); + } + + void Signer::process_bproof(rct::Bulletproof & bproof){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){ + if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){ + return nullptr; + } + + auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + compute_bproof(*(res->mutable_rsig_data())); + res->set_is_offloaded_bp(true); + return res; + } + + void Signer::step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ m_ct.cur_batch_idx += 1; m_ct.cur_output_in_batch_idx = 0; } @@ -814,12 +919,11 @@ namespace tx { res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); res->set_pseudo_out_alpha(m_ct.alphas[idx]); res->set_spend_key(m_ct.spend_encs[idx]); - if (!in_memory()){ - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); - res->set_pseudo_out(m_ct.pseudo_outs[idx]); - res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); - } + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); return res; } @@ -829,6 +933,19 @@ namespace tx { throw exc::ProtocolException("Cannot deserialize mg[i]"); } + // Sync updated pseudo_outputs, client_version>=1, HF10+ + if (client_version() >= 1 && ack->has_pseudo_out()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index"); + m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out(); + if (is_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->p.pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->p.pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } else { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } + } + m_ct.rv->p.MGs.push_back(mg); } @@ -841,14 +958,14 @@ namespace tx { if (m_multisig){ auto & cout_key = ack->cout_key(); for(auto & cur : m_ct.couts){ - if (cur.size() != 12 + 32){ + if (cur.size() != crypto::chacha::IV_SIZE + 32){ throw std::invalid_argument("Encrypted cout has invalid length"); } char buff[32]; auto data = cur.data(); - crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); + crypto::chacha::decrypt(data + crypto::chacha::IV_SIZE, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); m_ct.couts_dec.emplace_back(buff, 32); } } @@ -887,6 +1004,82 @@ namespace tx { return sb.GetString(); } + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data) + { + rapidjson::Document json; + + // The contents should be JSON if the wallet follows the new format. + if (json.Parse(data.c_str()).HasParseError()) + { + throw std::invalid_argument("Data parsing error"); + } + else if(!json.IsObject()) + { + throw std::invalid_argument("Data parsing error - not an object"); + } + + GET_FIELD_FROM_JSON(json, version, int, Int, true, -1); + GET_STRING_FROM_JSON(json, salt1, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, salt2, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, enc_keys, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, tx_prefix_hash, std::string, false, std::string()); + + if (field_version != 1) + { + throw std::invalid_argument("Unknown version"); + } + + res.salt1 = field_salt1; + res.salt2 = field_salt2; + res.tx_enc_keys = field_enc_keys; + res.tx_prefix_hash = field_tx_prefix_hash; + } + + std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data) + { + auto req = std::make_shared<messages::monero::MoneroGetTxKeyRequest>(); + req->set_salt1(tx_data.salt1); + req->set_salt2(tx_data.salt2); + req->set_tx_enc_keys(tx_data.tx_enc_keys); + req->set_tx_prefix_hash(tx_data.tx_prefix_hash); + req->set_reason(0); + + return req; + } + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack + ) + { + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, tx_prefix_hash, ack->salt()); + auto & encrypted_keys = ack->has_tx_derivations() ? ack->tx_derivations() : ack->tx_keys(); + + const size_t len_ciphertext = encrypted_keys.size(); // IV || keys || TAG + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t keys_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[keys_len]); + + protocol::crypto::chacha::decrypt( + encrypted_keys.data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast<const uint8_t *>(enc_key.data), + reinterpret_cast<const uint8_t *>(encrypted_keys.data()), + reinterpret_cast<char *>(plaintext.get()), &keys_len); + + CHECK_AND_ASSERT_THROW_MES(keys_len % 32 == 0, "Invalid size"); + tx_keys.resize(keys_len / 32); + + for(unsigned i = 0; i < keys_len / 32; ++i) + { + memcpy(tx_keys[i].data, plaintext.get() + 32 * i, 32); + } + memwipe(plaintext.get(), keys_len); + } } } diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index 23f94f948..98ca52565 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -92,11 +92,14 @@ namespace protocol{ // Crypto / encryption namespace crypto { namespace chacha { + // Constants as defined in RFC 7539. + const unsigned IV_SIZE = 12; + const unsigned TAG_SIZE = 16; // crypto_aead_chacha20poly1305_IETF_ABYTES; /** * Chacha20Poly1305 decryption with tag verification. RFC 7539. */ - void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext); + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len=nullptr); } } @@ -129,6 +132,14 @@ namespace ki { const std::vector<tools::wallet2::transfer_details> & transfers, std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + /** + * Processes Live refresh step response, parses KI, checks the signature + */ + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki); } // Cold transaction signing @@ -153,6 +164,7 @@ namespace tx { std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt); typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; @@ -164,8 +176,8 @@ namespace tx { TsxData tsx_data; tx_construction_data tx_data; cryptonote::transaction tx; - bool in_memory; unsigned rsig_type; + int bp_version; std::vector<uint64_t> grouping_vct; std::shared_ptr<MoneroRsigData> rsig_param; size_t cur_input_idx; @@ -206,6 +218,7 @@ namespace tx { const unsigned_tx_set * m_unsigned_tx; hw::tx_aux_data * m_aux_data; + unsigned m_client_version; bool m_multisig; const tx_construction_data & cur_tx(){ @@ -215,6 +228,9 @@ namespace tx { void extract_payment_id(); void compute_integrated_indices(TsxData * tsx_data); + bool should_compute_bp_now() const; + void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data); + void process_bproof(rct::Bulletproof & bproof); public: Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); @@ -238,6 +254,9 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx); void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_rsig(size_t idx); + void step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set(); void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev); @@ -249,8 +268,8 @@ namespace tx { std::string store_tx_aux_info(); - bool in_memory() const { - return m_ct.in_memory; + unsigned client_version() const { + return m_client_version; } bool is_simple() const { @@ -290,6 +309,18 @@ namespace tx { } }; + // TX Key decryption + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data); + + std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data); + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack + ); } } diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt new file mode 100644 index 000000000..1f369f114 --- /dev/null +++ b/src/lmdb/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(lmdb_sources database.cpp error.cpp table.cpp value_stream.cpp) +set(lmdb_headers database.h error.h key_stream.h table.h transaction.h util.h value_stream.h) + +monero_add_library(lmdb_lib ${lmdb_sources} ${lmdb_headers}) +target_link_libraries(lmdb_lib common ${LMDB_LIBRARY}) diff --git a/src/lmdb/database.cpp b/src/lmdb/database.cpp new file mode 100644 index 000000000..c6b244671 --- /dev/null +++ b/src/lmdb/database.cpp @@ -0,0 +1,187 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "database.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +#ifdef _WIN32 +namespace +{ + constexpr const mdb_mode_t open_flags = 0; +} +#else +#include <sys/stat.h> + +namespace +{ + constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR); +} +#endif + +namespace lmdb +{ + namespace + { + constexpr const std::size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB + void acquire_context(context& ctx) noexcept + { + while (ctx.lock.test_and_set()); + ++(ctx.active); + ctx.lock.clear(); + } + + void release_context(context& ctx) noexcept + { + --(ctx.active); + } + } + + void release_read_txn::operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + { + MDB_env* const env = mdb_txn_env(ptr); + abort_txn{}(ptr); + if (env) + { + context* ctx = reinterpret_cast<context*>(mdb_env_get_userctx(env)); + if (ctx) + release_context(*ctx); + } + } + } + + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept + { + MONERO_PRECOND(path != nullptr); + + MDB_env* obj = nullptr; + MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj))); + environment out{obj}; + + MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs)); + MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags)); + return {std::move(out)}; + } + + expect<write_txn> database::do_create_txn(unsigned int flags) noexcept + { + MONERO_PRECOND(handle() != nullptr); + + for (unsigned attempts = 0; attempts < 3; ++attempts) + { + acquire_context(ctx); + + MDB_txn* txn = nullptr; + const int err = + mdb_txn_begin(handle(), nullptr, flags, &txn); + if (!err && txn != nullptr) + return write_txn{txn}; + + release_context(ctx); + if (err != MDB_MAP_RESIZED) + return {lmdb::error(err)}; + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_RESIZED)}; + } + + database::database(environment env) + : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT} + { + if (handle()) + { + const int err = mdb_env_set_userctx(handle(), std::addressof(ctx)); + if (err) + MONERO_THROW(lmdb::error(err), "Failed to set user context"); + } + } + + database::~database() noexcept + { + while (ctx.active); + } + + expect<void> database::resize() noexcept + { + MONERO_PRECOND(handle() != nullptr); + + while (ctx.lock.test_and_set()); + while (ctx.active); + + MDB_envinfo info{}; + MONERO_LMDB_CHECK(mdb_env_info(handle(), &info)); + + const std::size_t resize = std::min(info.me_mapsize, max_resize); + const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize); + ctx.lock.clear(); + if (err) + return {lmdb::error(err)}; + return success(); + } + + expect<read_txn> database::create_read_txn(suspended_txn txn) noexcept + { + if (txn) + { + acquire_context(ctx); + const int err = mdb_txn_renew(txn.get()); + if (err) + { + release_context(ctx); + return {lmdb::error(err)}; + } + return read_txn{txn.release()}; + } + auto new_txn = do_create_txn(MDB_RDONLY); + if (new_txn) + return read_txn{new_txn->release()}; + return new_txn.error(); + } + + expect<suspended_txn> database::reset_txn(read_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + mdb_txn_reset(txn.get()); + release_context(ctx); + return suspended_txn{txn.release()}; + } + + expect<write_txn> database::create_write_txn() noexcept + { + return do_create_txn(0); + } + + expect<void> database::commit(write_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + MONERO_LMDB_CHECK(mdb_txn_commit(txn.get())); + txn.release(); + release_context(ctx); + return success(); + } +} // lmdb diff --git a/src/lmdb/database.h b/src/lmdb/database.h new file mode 100644 index 000000000..269f8c8a1 --- /dev/null +++ b/src/lmdb/database.h @@ -0,0 +1,138 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <atomic> +#include <cstddef> +#include <lmdb.h> +#include <memory> +#include <type_traits> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/transaction.h" + +namespace lmdb +{ + //! Closes LMDB environment handle. + struct close_env + { + void operator()(MDB_env* ptr) const noexcept + { + if (ptr) + mdb_env_close(ptr); + } + }; + + using environment = std::unique_ptr<MDB_env, close_env>; + + //! \return LMDB environment at `path` with a max of `max_dbs` tables. + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept; + + //! Context given to LMDB. + struct context + { + std::atomic<std::size_t> active; + std::atomic_flag lock; + }; + + //! Manages a LMDB environment for safe memory-map resizing. Thread-safe. + class database + { + environment env; + context ctx; + + //! \return The LMDB environment associated with the object. + MDB_env* handle() const noexcept { return env.get(); } + + expect<write_txn> do_create_txn(unsigned int flags) noexcept; + + public: + database(environment env); + + database(database&&) = delete; + database(database const&) = delete; + + virtual ~database() noexcept; + + database& operator=(database&&) = delete; + database& operator=(database const&) = delete; + + /*! + Resize the memory map for the LMDB environment. Will block until + all reads/writes on the environment complete. + */ + expect<void> resize() noexcept; + + //! \return A read only LMDB transaction, reusing `txn` if provided. + expect<read_txn> create_read_txn(suspended_txn txn = nullptr) noexcept; + + //! \return `txn` after releasing context. + expect<suspended_txn> reset_txn(read_txn txn) noexcept; + + //! \return A read-write LMDB transaction. + expect<write_txn> create_write_txn() noexcept; + + //! Commit the read-write transaction. + expect<void> commit(write_txn txn) noexcept; + + /*! + Create a write transaction, pass it to `f`, then try to commit + the write if `f` succeeds. + + \tparam F must be callable with signature `expect<T>(MDB_txn&)`. + \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`. + + \return The result of calling `f`. + */ + template<typename F> + typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3) + { + for (unsigned i = 0; i < attempts; ++i) + { + expect<write_txn> txn = create_write_txn(); + if (!txn) + return txn.error(); + + MONERO_PRECOND(*txn != nullptr); + const auto wrote = f(*(*txn)); + if (wrote) + { + MONERO_CHECK(commit(std::move(*txn))); + return wrote; + } + if (wrote != lmdb::error(MDB_MAP_FULL)) + return wrote; + + txn->reset(); + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_FULL)}; + } + }; +} // lmdb + diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp new file mode 100644 index 000000000..359677064 --- /dev/null +++ b/src/lmdb/error.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include <lmdb.h> +#include <string> + +namespace { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "lmdb::error_category()"; + } + + virtual std::string message(int value) const override final + { + char const* const msg = mdb_strerror(value); + if (msg) + return msg; + return "Unknown lmdb::error_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + switch (value) + { + case MDB_KEYEXIST: + case MDB_NOTFOUND: + break; // map to nothing generic + case MDB_PAGE_NOTFOUND: + case MDB_CORRUPTED: + return std::errc::state_not_recoverable; + case MDB_PANIC: + case MDB_VERSION_MISMATCH: + case MDB_INVALID: + break; // map to nothing generic + case MDB_MAP_FULL: + return std::errc::no_buffer_space; + case MDB_DBS_FULL: + break; // map to nothing generic + case MDB_READERS_FULL: + case MDB_TLS_FULL: + return std::errc::no_lock_available; + case MDB_TXN_FULL: + case MDB_CURSOR_FULL: + case MDB_PAGE_FULL: + case MDB_MAP_RESIZED: + break; // map to nothing generic + case MDB_INCOMPATIBLE: + return std::errc::invalid_argument; + case MDB_BAD_RSLOT: + case MDB_BAD_TXN: + case MDB_BAD_VALSIZE: + case MDB_BAD_DBI: + return std::errc::invalid_argument; + default: + return std::error_condition{value, std::generic_category()}; + } + return std::error_condition{value, *this}; + } + }; +} + +namespace lmdb +{ + std::error_category const& error_category() noexcept + { + static const category instance{}; + return instance; + } +} + diff --git a/src/lmdb/error.h b/src/lmdb/error.h new file mode 100644 index 000000000..2944adf78 --- /dev/null +++ b/src/lmdb/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <system_error> +#include <type_traits> + +//! Executes a LMDB command, and returns errors via `lmdb::error` enum. +#define MONERO_LMDB_CHECK(...) \ + do \ + { \ + const int err = __VA_ARGS__ ; \ + if (err) \ + return {lmdb::error(err)}; \ + } while (0) + +namespace lmdb +{ + //! Tracks LMDB error codes. + enum class error : int + { + // 0 is reserved for no error, as per expect<T> + // All other errors are the values reported by LMDB + }; + + std::error_category const& error_category() noexcept; + + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::lmdb::error> + : true_type + {}; +} diff --git a/src/lmdb/key_stream.h b/src/lmdb/key_stream.h new file mode 100644 index 000000000..40434d3a1 --- /dev/null +++ b/src/lmdb/key_stream.h @@ -0,0 +1,264 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "lmdb/value_stream.h" +#include "span.h" + +namespace lmdb +{ + + /*! + An InputIterator for a fixed-sized LMDB key and value. `operator++` + iterates over keys. + + \tparam K Key type in database records. + \tparam V Value type in database records. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename K, typename V> + class key_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> key; + + void increment() + { + // MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/ + if (cur) + key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first; + } + + public: + using value_type = std::pair<K, boost::iterator_range<value_iterator<V>>>; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + key_iterator() noexcept + : cur(nullptr), key() + {} + + /*! + \param cur Iterate over keys starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + key_iterator(MDB_cursor* cur) + : cur(cur), key() + { + if (cur) + key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first; + } + + //! \return True if `this` is one-past the last key. + bool is_end() const noexcept { return key.empty(); } + + //! \return True iff `rhs` is referencing `this` key. + bool equal(key_iterator const& rhs) const noexcept + { + return + (key.empty() && rhs.key.empty()) || + key.data() == rhs.key.data(); + } + + /*! + Moves iterator to next key or end. Invalidates all prior copies of + the iterator. + */ + key_iterator& operator++() + { + increment(); + return *this; + } + + /*! + Moves iterator to next key or end. + + \return A copy that is already invalidated, ignore + */ + key_iterator operator++(int) + { + key_iterator out{*this}; + increment(); + return out; + } + + //! \pre `!is_end()` \return {current key, current value range} + value_type operator*() const + { + return {get_key(), make_value_range()}; + } + + //! \pre `!is_end()` \return Current key + K get_key() const noexcept + { + assert(!is_end()); + K out; + std::memcpy(std::addressof(out), key.data(), sizeof(out)); + return out; + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return an entire value (`V`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + value_iterator<T, F, offset> make_value_iterator() const + { + static_assert(std::is_same<T, V>(), "bad MONERO_FIELD usage?"); + return {cur}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + boost::iterator_range<value_iterator<T, F, offset>> make_value_range() const + { + return {make_value_iterator<T, F, offset>(), value_iterator<T, F, offset>{}}; + } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and + value `V`. + + \tparam K key type being stored by each record. + \tparam V value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename K, typename V, typename D> + class key_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit key_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + key_stream(key_stream&&) = default; + key_stream(key_stream const&) = delete; + ~key_stream() = default; + key_stream& operator=(key_stream&&) = default; + key_stream& operator=(key_stream const&) = delete; + + /*! + Give up ownership of the cursor. `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first key/value. Newly created + iterators will start at the first value again. + + \note Invalidates all current iterators, including those created + with `make_iterator` or `make_range`. Also invalidates all + `value_iterator`s created with `key_iterator`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator over database keys from current cursor + position that will reach `.is_end()` after the last key. + */ + key_iterator<K, V> make_iterator() const + { + return {cur.get()}; + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Range from current cursor position until last key record. + Useful in for-each range loops or in templated code + */ + boost::iterator_range<key_iterator<K, V>> make_range() const + { + return {make_iterator(), key_iterator<K, V>{}}; + } + }; + + template<typename K, typename V> + inline + bool operator==(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename K, typename V> + inline + bool operator!=(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/lmdb/table.cpp b/src/lmdb/table.cpp new file mode 100644 index 000000000..0818b74e6 --- /dev/null +++ b/src/lmdb/table.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "table.h" + +namespace lmdb +{ + expect<MDB_dbi> table::open(MDB_txn& write_txn) const noexcept + { + MONERO_PRECOND(name != nullptr); + + MDB_dbi out; + MONERO_LMDB_CHECK(mdb_dbi_open(&write_txn, name, flags, &out)); + if (key_cmp && !(flags & MDB_INTEGERKEY)) + MONERO_LMDB_CHECK(mdb_set_compare(&write_txn, out, key_cmp)); + if (value_cmp && !(flags & MDB_INTEGERDUP)) + MONERO_LMDB_CHECK(mdb_set_dupsort(&write_txn, out, value_cmp)); + return out; + } +} diff --git a/src/lmdb/table.h b/src/lmdb/table.h new file mode 100644 index 000000000..41a3de296 --- /dev/null +++ b/src/lmdb/table.h @@ -0,0 +1,120 @@ +#pragma once + +#include <utility> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/key_stream.h" +#include "lmdb/util.h" +#include "lmdb/value_stream.h" + +namespace lmdb +{ + //! Helper for grouping typical LMDB DBI options. + struct table + { + char const* const name; + const unsigned flags; + MDB_cmp_func const* const key_cmp; + MDB_cmp_func const* const value_cmp; + + //! \pre `name != nullptr` \return Open table. + expect<MDB_dbi> open(MDB_txn& write_txn) const noexcept; + }; + + //! Helper for grouping typical LMDB DBI options when key and value are fixed types. + template<typename K, typename V> + struct basic_table : table + { + using key_type = K; + using value_type = V; + + //! \return Additional LMDB flags based on `flags` value. + static constexpr unsigned compute_flags(const unsigned flags) noexcept + { + return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0); + } + + constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : table{name, compute_flags(flags), &lmdb::less<lmdb::native_type<K>>, value_cmp} + {} + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template<typename U, typename F = U, std::size_t offset = 0> + static expect<F> get_value(MDB_val value) noexcept + { + static_assert(std::is_same<U, V>(), "bad MONERO_FIELD?"); + static_assert(std::is_pod<F>(), "F must be POD"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size != sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast<char*>(value.mv_data) + offset, sizeof(out)); + return out; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first key/value in the table linked + to `cur` or an empty `key_stream`. + */ + template<typename D> + expect<key_stream<K, V, D>> + static get_key_stream(std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key; + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return key_stream<K, V, D>{std::move(cur)}; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first value at `key` in the table linked + to `cur` or an empty `value_stream`. + */ + template<typename D> + expect<value_stream<V, D>> + static get_value_stream(K const& key, std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return value_stream<V, D>{std::move(cur)}; + } + }; +} // lmdb + diff --git a/src/lmdb/transaction.h b/src/lmdb/transaction.h new file mode 100644 index 000000000..cdd80696c --- /dev/null +++ b/src/lmdb/transaction.h @@ -0,0 +1,95 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <lmdb.h> +#include <memory> + +#include "lmdb/error.h" + +//! Uses C++ type system to differentiate between cursors +#define MONERO_CURSOR(name) \ + struct close_ ## name : ::lmdb::close_cursor {}; \ + using name = std::unique_ptr< MDB_cursor, close_ ## name >; + +namespace lmdb +{ + struct abort_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + mdb_txn_abort(ptr); + } + }; + + /*! + Only valid if used via `create_read_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct release_read_txn + { + void operator()(MDB_txn* ptr) const noexcept; + // implementation in database.cpp + }; + + /*! + Only valid if used via `create_write_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct abort_write_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + release_read_txn{}(ptr); + } + }; + + struct close_cursor + { + void operator()(MDB_cursor* ptr) const noexcept + { + if (ptr) + mdb_cursor_close(ptr); + } + }; + + template<typename D> + inline expect<std::unique_ptr<MDB_cursor, D>> + open_cursor(MDB_txn& txn, MDB_dbi tbl) noexcept + { + MDB_cursor* cur = nullptr; + MONERO_LMDB_CHECK(mdb_cursor_open(&txn, tbl, &cur)); + return std::unique_ptr<MDB_cursor, D>{cur}; + } + + // The below use the C++ type system to designate `MDB_txn` status. + + using suspended_txn = std::unique_ptr<MDB_txn, abort_txn>; + using read_txn = std::unique_ptr<MDB_txn, release_read_txn>; + using write_txn = std::unique_ptr<MDB_txn, abort_write_txn>; +} // lmdb diff --git a/src/lmdb/util.h b/src/lmdb/util.h new file mode 100644 index 000000000..50162b7c8 --- /dev/null +++ b/src/lmdb/util.h @@ -0,0 +1,149 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <cstddef> +#include <cstring> +#include <lmdb.h> +#include <type_traits> +#include <utility> + +#include "span.h" + +/*! Calculates types and offset of struct field. Use in template arguments for + `table::get_value`, `value_iterator::get_value`, + `value_stream::make_iterator`, or `value_stream::make_range`. */ +#define MONERO_FIELD(obj, field) \ + obj , decltype(std::declval<obj>().field) , offsetof(obj, field) + +//! Expands to `lmdb::less` for the value `field` within `obj`. +#define MONERO_SORT_BY(obj, field) \ + &::lmdb::less< \ + lmdb::native_type<decltype(std::declval<obj>().field)>, \ + offsetof(obj, field) \ + > + +//! Expands to `lmdb::compare` for the value `field` within `obj`. +#define MONERO_COMPARE(obj, field) \ + &::lmdb::compare< \ + decltype(std::declval<obj>().field), \ + offsetof(obj, field) \ + > + +namespace lmdb +{ + //! Prevent instantiation of `std::underlying_type<T>` when `T` is not enum. + template<typename T> + struct identity + { + using type = T; + }; + + /*! + Get the native type for enums, or return `T` unchanged. Useful for + merging generated machine code for templated functions that use enums + with identical size-widths without relying on aggressive identical + comdat folding (ICF) support in linker. So with enum defintion + `enum class enum_foo : unsigned long {};` will always yield + `assert(&func_foo<unsigned long> == &func_foo<native_type<enum_foo>>)`. + */ + template<typename T> + using native_type = typename std::conditional< + std::is_enum<T>::value, std::underlying_type<T>, identity<T> + >::type::type; + + //! \return `value` as its native type. + template<typename T, typename U = typename std::underlying_type<T>::type> + inline constexpr U to_native(T value) noexcept + { + return U(value); + } + + //! \return `value` bytes in a LMDB `MDB_val` object. + template<typename T> + inline MDB_val to_val(T&& value) noexcept + { + // lmdb does not touch user data, so const_cast is acceptable + static_assert(!std::is_rvalue_reference<T&&>(), "cannot use temporary value"); + void const* const temp = reinterpret_cast<void const*>(std::addressof(value)); + return MDB_val{sizeof(value), const_cast<void*>(temp)}; + } + + //! \return A span over the same chunk of memory as `value`. + inline constexpr epee::span<const std::uint8_t> to_byte_span(MDB_val value) noexcept + { + return {static_cast<const std::uint8_t*>(value.mv_data), value.mv_size}; + } + + /*! + A LMDB comparison function that uses `operator<`. + + \tparam T has a defined `operator<` . + \tparam offset to `T` within the value. + + \return -1 if `left < right`, 1 if `right < left`, and 0 otherwise. + */ + template<typename T, std::size_t offset = 0> + inline int less(MDB_val const* left, MDB_val const* right) noexcept + { + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + + T left_val; + T right_val; + std::memcpy(std::addressof(left_val), static_cast<char*>(left->mv_data) + offset, sizeof(T)); + std::memcpy(std::addressof(right_val), static_cast<char*>(right->mv_data) + offset, sizeof(T)); + return left_val < right_val ? -1 : bool(right_val < left_val); + } + + /*! + A LMDB comparison function that uses `std::memcmp`. + + \toaram T is `!epee::has_padding` + \tparam offset to `T` within the value. + + \return The result of `std::memcmp` over the value. + */ + template<typename T, std::size_t offset = 0> + inline int compare(MDB_val const* left, MDB_val const* right) noexcept + { + static_assert(!epee::has_padding<T>(), "memcmp will not work"); + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + return std::memcmp( + static_cast<char*>(left->mv_data) + offset, + static_cast<char*>(right->mv_data) + offset, + sizeof(T) + ); + } +} // lmdb diff --git a/src/lmdb/value_stream.cpp b/src/lmdb/value_stream.cpp new file mode 100644 index 000000000..1024deb06 --- /dev/null +++ b/src/lmdb/value_stream.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "value_stream.h" + +#include <stdexcept> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +namespace lmdb +{ + namespace stream + { + std::size_t count(MDB_cursor* cur) + { + std::size_t out = 0; + if (cur) + { + const int rc = mdb_cursor_count(cur, &out); + if (rc) + MONERO_THROW(lmdb::error(rc), "mdb_cursor_count"); + } + return out; + } + + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value) + { + MDB_val key_bytes{}; + MDB_val value_bytes{}; + const int rc = mdb_cursor_get(&cur, &key_bytes, &value_bytes, op); + if (rc) + { + if (rc == MDB_NOTFOUND) + return {}; + MONERO_THROW(lmdb::error(rc), "mdb_cursor_get"); + } + + if (key && key != key_bytes.mv_size) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get key"); + + if (value && (value_bytes.mv_size % value != 0 || value_bytes.mv_size == 0)) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get value"); + + return {lmdb::to_byte_span(key_bytes), lmdb::to_byte_span(value_bytes)}; + } + } +} + diff --git a/src/lmdb/value_stream.h b/src/lmdb/value_stream.h new file mode 100644 index 000000000..c9977221f --- /dev/null +++ b/src/lmdb/value_stream.h @@ -0,0 +1,287 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "span.h" + +namespace lmdb +{ + namespace stream + { + /* + \throw std::system_error if unexpected LMDB error. + \return 0 if `cur == nullptr`, otherwise count of values at current key. + */ + std::size_t count(MDB_cursor* cur); + + /*! + Calls `mdb_cursor_get` and does some error checking. + + \param cur is given to `mdb_cursor_get` without modification. + \param op is passed to `mdb_cursor_get` without modification. + \param key expected key size or 0 to skip key size check. + \param value expected value size or 0 to skip value size check. + + \throw std::system_error if `key != 0` and `key_.mv_size != key`. + \throw std::system_error if `value != 0` and `value_.mv_size != value`. + \throw std::system_error if `mdb_cursor_get` returns any error + other than `MDB_NOTFOUND`. + + \return {key bytes, value bytes} or two empty spans if `MDB_NOTFOUND`. + */ + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value); + } + + /*! + An InputIterator for a fixed-sized LMDB value at a specific key. + + \tparam T The value type at the specific key. + \tparam F The value type being returned when dereferenced. + \tparam offset to `F` within `T`. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename T, typename F = T, std::size_t offset = 0> + class value_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> values; + + void increment() + { + values.remove_prefix(sizeof(T)); + if (values.empty() && cur) + values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second; + } + + public: + using value_type = F; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + value_iterator() noexcept + : cur(nullptr), values() + {} + + /*! + \param cur Iterate over values starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + value_iterator(MDB_cursor* cur) + : cur(cur), values() + { + if (cur) + values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second; + } + + value_iterator(value_iterator const&) = default; + ~value_iterator() = default; + value_iterator& operator=(value_iterator const&) = default; + + //! \return True if `this` is one-past the last value. + bool is_end() const noexcept { return values.empty(); } + + //! \return True iff `rhs` is referencing `this` value. + bool equal(value_iterator const& rhs) const noexcept + { + return + (values.empty() && rhs.values.empty()) || + values.data() == rhs.values.data(); + } + + //! Invalidates all prior copies of the iterator. + value_iterator& operator++() + { + increment(); + return *this; + } + + //! \return A copy that is already invalidated, ignore + value_iterator operator++(int) + { + value_iterator out{*this}; + increment(); + return out; + } + + /*! + Get a specific field within `F`. Default behavior is to return + the entirety of `U`, despite the filtering logic of `operator*`. + + \pre `!is_end()` + + \tparam U must match `T`, used for `MONERO_FIELD` sanity checking. + \tparam G field type to extract from the value + \tparam uoffset to `G` type, or `0` when `std::is_same<U, G>()`. + + \return The field `G`, at `uoffset` within `U`. + */ + template<typename U, typename G = U, std::size_t uoffset = 0> + G get_value() const noexcept + { + static_assert(std::is_same<U, T>(), "bad MONERO_FIELD usage?"); + static_assert(std::is_pod<U>(), "value type must be pod"); + static_assert(std::is_pod<G>(), "field type must be pod"); + static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset"); + assert(sizeof(G) + uoffset <= values.size()); + assert(!is_end()); + + G value; + std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value)); + return value; + } + + //! \pre `!is_end()` \return The field `F`, at `offset`, within `T`. + value_type operator*() const noexcept { return get_value<T, F, offset>(); } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`. + + \tparam T value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename T, typename D> + class value_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit value_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + value_stream(value_stream&&) = default; + value_stream(value_stream const&) = delete; + ~value_stream() = default; + value_stream& operator=(value_stream&&) = default; + value_stream& operator=(value_stream const&) = delete; + + /*! + Give up ownership of the cursor. `count()`, `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first value. Newly created iterators + will start at the first value again. + + \note Invalidates all current iterators from `this`, including + those created with `make_iterator` or `make_range`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Number of values at this key. + */ + std::size_t count() const + { + return lmdb::stream::count(cur.get()); + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return entire value (`T`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + value_iterator<U, F, offset> make_iterator() const + { + static_assert(std::is_same<U, T>(), "was MONERO_FIELD used with wrong type?"); + return {cur.get()}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + boost::iterator_range<value_iterator<U, F, offset>> make_range() const + { + return {make_iterator<U, F, offset>(), value_iterator<U, F, offset>{}}; + } + }; + + template<typename T, typename F, std::size_t offset> + inline + bool operator==(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename T, typename F, std::size_t offset> + inline + bool operator!=(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index ff55167fe..56b0361a7 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1471,7 +1471,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get last block."; return false; } - bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1513,7 +1514,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1563,7 +1565,8 @@ namespace cryptonote return false; } res.headers.push_back(block_header_response()); - bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1597,7 +1600,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get block by height. Height = " + std::to_string(req.height) + '.'; return false; } - bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1652,7 +1656,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 3c89c68a0..f390b7675 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -2049,7 +2049,7 @@ bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux); // import key images - return m_wallet->import_key_images(exported_txs.key_images); + return m_wallet->import_key_images(exported_txs, 0, true); } bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -4686,12 +4686,12 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_button_request() +void simple_wallet::on_device_button_request(uint64_t code) { message_writer(console_color_white, false) << tr("Device requires attention"); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request() { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4699,14 +4699,14 @@ void simple_wallet::on_pin_request(epee::wipeable_string & pin) std::string msg = tr("Enter device PIN"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN")); - pin = pwd_container->password(); + return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device) { if (on_device){ message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); - return; + return boost::none; } #ifdef HAVE_READLINE @@ -4715,13 +4715,13 @@ void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string std::string msg = tr("Enter device passphrase"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); - passphrase = pwd_container->password(); + return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money) { // Key image sync after the first refresh - if (!m_wallet->get_account().get_device().has_tx_cold_sign()) { + if (!m_wallet->get_account().get_device().has_tx_cold_sign() || m_wallet->get_account().get_device().has_ki_live_refresh()) { return; } @@ -6772,7 +6772,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) { std::vector<std::string> local_args = args_; - if (m_wallet->key_on_device()) + if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; @@ -6793,7 +6793,9 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; - if (m_wallet->get_tx_key(txid, tx_key, additional_tx_keys)) + + bool found_tx_key = m_wallet->get_tx_key(txid, tx_key, additional_tx_keys); + if (found_tx_key) { ostringstream oss; oss << epee::string_tools::pod_to_hex(tx_key); @@ -6869,7 +6871,7 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) { - if (m_wallet->key_on_device()) + if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; @@ -8655,13 +8657,8 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) { uint64_t spent = 0, unspent = 0; uint64_t height = m_wallet->import_key_images(filename, spent, unspent); - if (height > 0) - { - success_msg_writer() << "Signed key images imported to height " << height << ", " - << print_money(spent) << " spent, " << print_money(unspent) << " unspent"; - } else { - fail_msg_writer() << "Failed to import key images"; - } + success_msg_writer() << "Signed key images imported to height " << height << ", " + << print_money(spent) << " spent, " << print_money(unspent) << " unspent"; } catch (const std::exception &e) { @@ -10012,6 +10009,16 @@ bool simple_wallet::mms(const std::vector<std::string> &args) { try { + m_wallet->get_multisig_wallet_state(); + } + catch(const std::exception &e) + { + fail_msg_writer() << tr("MMS not available in this wallet"); + return true; + } + + try + { mms::message_store& ms = m_wallet->get_message_store(); if (args.size() == 0) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index cf5458175..7bcb92190 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -300,9 +300,9 @@ namespace cryptonote virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); - virtual void on_button_request(); - virtual void on_pin_request(epee::wipeable_string & pin); - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + virtual void on_device_button_request(uint64_t code); + virtual boost::optional<epee::wipeable_string> on_device_pin_request(); + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); //---------------------------------------------------------- friend class refresh_progress_reporter_t; diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index e649f1f3a..52510164a 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -109,6 +109,23 @@ bool PendingTransactionImpl::commit(const std::string &filename, bool overwrite) } m_wallet.pauseRefresh(); + + const bool tx_cold_signed = m_wallet.m_wallet->get_account().get_device().has_tx_cold_sign(); + if (tx_cold_signed){ + std::unordered_set<size_t> selected_transfers; + for(const tools::wallet2::pending_tx & ptx : m_pending_tx){ + for(size_t s : ptx.selected_transfers){ + selected_transfers.insert(s); + } + } + + m_wallet.m_wallet->cold_tx_aux_import(m_pending_tx, m_tx_device_aux); + bool r = m_wallet.m_wallet->import_key_images(m_key_images, 0, selected_transfers); + if (!r){ + throw runtime_error("Cold sign transaction submit failed - key image sync fail"); + } + } + while (!m_pending_tx.empty()) { auto & ptx = m_pending_tx.back(); m_wallet.m_wallet->commit_tx(ptx); diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index 4ec7c656a..92801d77d 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -67,6 +67,8 @@ private: std::string m_errorString; std::vector<tools::wallet2::pending_tx> m_pending_tx; std::unordered_set<crypto::public_key> m_signers; + std::vector<std::string> m_tx_device_aux; + std::vector<crypto::key_image> m_key_images; }; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index da6ddc8a3..d7226b656 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -242,6 +242,42 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback } } + virtual void on_device_button_request(uint64_t code) + { + if (m_listener) { + m_listener->onDeviceButtonRequest(code); + } + } + + virtual boost::optional<epee::wipeable_string> on_device_pin_request() + { + if (m_listener) { + auto pin = m_listener->onDevicePinRequest(); + if (pin){ + return boost::make_optional(epee::wipeable_string((*pin).data(), (*pin).size())); + } + } + return boost::none; + } + + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) + { + if (m_listener) { + auto passphrase = m_listener->onDevicePassphraseRequest(on_device); + if (!on_device && passphrase) { + return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size())); + } + } + return boost::none; + } + + virtual void on_device_progress(const hw::device_progress & event) + { + if (m_listener) { + m_listener->onDeviceProgress(DeviceProgress(event.progress(), event.indeterminate())); + } + } + WalletListener * m_listener; WalletImpl * m_wallet; }; @@ -785,6 +821,28 @@ bool WalletImpl::setPassword(const std::string &password) return status() == Status_Ok; } +bool WalletImpl::setDevicePin(const std::string &pin) +{ + clearStatus(); + try { + m_wallet->get_account().get_device().set_pin(epee::wipeable_string(pin.data(), pin.size())); + } catch (const std::exception &e) { + setStatusError(e.what()); + } + return status() == Status_Ok; +} + +bool WalletImpl::setDevicePassphrase(const std::string &passphrase) +{ + clearStatus(); + try { + m_wallet->get_account().get_device().set_passphrase(epee::wipeable_string(passphrase.data(), passphrase.size())); + } catch (const std::exception &e) { + setStatusError(e.what()); + } + return status() == Status_Ok; +} + std::string WalletImpl::address(uint32_t accountIndex, uint32_t addressIndex) const { return m_wallet->get_subaddress_as_str({accountIndex, addressIndex}); @@ -1428,8 +1486,12 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const extra, subaddr_account, subaddr_indices); } + pendingTxPostProcess(transaction); + if (multisig().isMultisig) { - transaction->m_signers = m_wallet->make_multisig_tx_set(transaction->m_pending_tx).m_signers; + auto tx_set = m_wallet->make_multisig_tx_set(transaction->m_pending_tx); + transaction->m_pending_tx = tx_set.m_ptx; + transaction->m_signers = tx_set.m_signers; } } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1511,6 +1573,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); + pendingTxPostProcess(transaction); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -2093,6 +2156,21 @@ bool WalletImpl::isNewWallet() const return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly(); } +void WalletImpl::pendingTxPostProcess(PendingTransactionImpl * pending) +{ + // If the device being used is HW device with cold signing protocol, cold sign then. + if (!m_wallet->get_account().get_device().has_tx_cold_sign()){ + return; + } + + tools::wallet2::signed_tx_set exported_txs; + std::vector<cryptonote::address_parse_info> dsts_info; + + m_wallet->cold_sign_tx(pending->m_pending_tx, exported_txs, dsts_info, pending->m_tx_device_aux); + pending->m_key_images = exported_txs.key_images; + pending->m_pending_tx = exported_txs.ptx; +} + bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) { // claim RPC so there's no in-memory encryption for now @@ -2325,6 +2403,11 @@ bool WalletImpl::isKeysFileLocked() { return m_wallet->is_keys_file_locked(); } + +uint64_t WalletImpl::coldKeyImageSync(uint64_t &spent, uint64_t &unspent) +{ + return m_wallet->cold_key_image_sync(spent, unspent); +} } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index bd33b773c..9e07b6e19 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -89,6 +89,8 @@ public: std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; bool setPassword(const std::string &password) override; + bool setDevicePin(const std::string &password) override; + bool setDevicePassphrase(const std::string &password) override; std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; std::string integratedAddress(const std::string &payment_id) const override; std::string secretViewKey() const override; @@ -198,6 +200,7 @@ public: virtual bool lockKeysFile() override; virtual bool unlockKeysFile() override; virtual bool isKeysFileLocked() override; + virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) override; private: void clearStatus() const; @@ -209,6 +212,7 @@ private: bool daemonSynced() const; void stopRefresh(); bool isNewWallet() const; + void pendingTxPostProcess(PendingTransactionImpl * pending); bool doInit(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false); private: diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index c549c260b..ee1d6ae79 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -324,6 +324,19 @@ struct MultisigState { uint32_t total; }; + +struct DeviceProgress { + DeviceProgress(): m_progress(0), m_indeterminate(false) {} + DeviceProgress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate) {} + + virtual double progress() const { return m_progress; } + virtual bool indeterminate() const { return m_indeterminate; } + +protected: + double m_progress; + bool m_indeterminate; +}; + struct WalletListener { virtual ~WalletListener() = 0; @@ -364,6 +377,31 @@ struct WalletListener * @brief refreshed - called when wallet refreshed by background thread or explicitly refreshed by calling "refresh" synchronously */ virtual void refreshed() = 0; + + /** + * @brief called by device if the action is required + */ + virtual void onDeviceButtonRequest(uint64_t code) {} + + /** + * @brief called by device when PIN is needed + */ + virtual optional<std::string> onDevicePinRequest() { + throw std::runtime_error("Not supported"); + } + + /** + * @brief called by device when passphrase entry is needed + */ + virtual optional<std::string> onDevicePassphraseRequest(bool on_device) { + if (!on_device) throw std::runtime_error("Not supported"); + return optional<std::string>(); + } + + /** + * @brief Signalizes device operation progress + */ + virtual void onDeviceProgress(const DeviceProgress & event) {}; }; @@ -375,7 +413,8 @@ struct Wallet { enum Device { Device_Software = 0, - Device_Ledger = 1 + Device_Ledger = 1, + Device_Trezor = 2 }; enum Status { @@ -401,6 +440,8 @@ struct Wallet //! returns both error and error string atomically. suggested to use in instead of status() and errorString() virtual void statusWithErrorString(int& status, std::string& errorString) const = 0; virtual bool setPassword(const std::string &password) = 0; + virtual bool setDevicePin(const std::string &password) { return false; }; + virtual bool setDevicePassphrase(const std::string &password) { return false; }; virtual std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const = 0; std::string mainAddress() const { return address(0, 0); } virtual std::string path() const = 0; @@ -947,6 +988,9 @@ struct Wallet * \return Device they are on */ virtual Device getDeviceType() const = 0; + + //! cold-device protocol key image sync + virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) = 0; }; /** diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 0cedd4b50..a46da42db 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -928,22 +928,30 @@ wallet_keys_unlocker::~wallet_keys_unlocker() } } -void wallet_device_callback::on_button_request() +void wallet_device_callback::on_button_request(uint64_t code) { if (wallet) - wallet->on_button_request(); + wallet->on_device_button_request(code); } -void wallet_device_callback::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request() { if (wallet) - wallet->on_pin_request(pin); + return wallet->on_device_pin_request(); + return boost::none; } -void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device) { if (wallet) - wallet->on_passphrase_request(on_device, passphrase); + return wallet->on_device_passphrase_request(on_device); + return boost::none; +} + +void wallet_device_callback::on_progress(const hw::device_progress& event) +{ + if (wallet) + wallet->on_device_progress(event); } wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): @@ -2883,6 +2891,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo std::vector<parsed_block> parsed_blocks; bool refreshed = false; std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> output_tracker_cache; + hw::device &hwdev = m_account.get_device(); // pull the first set of blocks get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); @@ -3039,6 +3048,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo LOG_PRINT_L1("Failed to check pending transactions"); } + hwdev.computing_key_images(false); m_first_refresh_done = true; LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); @@ -5110,7 +5120,14 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass MERROR("Failed to save rings, will try again next time"); } - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + try + { + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + } + catch (const std::exception &e) + { + MERROR("Failed to initialize MMS, it will be unusable"); + } } //---------------------------------------------------------------------------------------------------- void wallet2::trim_hashchain() @@ -7050,6 +7067,17 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; + // check the keys are valid + if (!rct::isInMainSubgroup(rct::pk2rct(output_public_key))) + { + MWARNING("Key " << output_public_key << " at index " << global_index << " is not in the main subgroup"); + return false; + } + if (!rct::isInMainSubgroup(mask)) + { + MWARNING("Commitment " << mask << " at index " << global_index << " is not in the main subgroup"); + return false; + } // if (is_output_blackballed(output_public_key)) // don't add blackballed outputs // return false; outs.back().push_back(item); @@ -8118,6 +8146,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry else { change_dts.addr = get_subaddress({subaddr_account, 0}); + change_dts.is_subaddress = subaddr_account != 0; splitted_dsts.push_back(change_dts); } @@ -8829,6 +8858,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp boost::unique_lock<hw::device> hwdev_lock (hwdev); hw::reset_mode rst(hwdev); + auto original_dsts = dsts; + if(m_light_wallet) { // Populate m_transfers light_wallet_get_unspent_outs(); @@ -9349,10 +9380,77 @@ skip_tx: ptx_vector.push_back(tx.ptx); } + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, original_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + // if we made it this far, we're OK to actually send the transactions return ptx_vector; } +bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const +{ + MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations"); + + hw::device &hwdev = m_account.get_device(); + + THROW_WALLET_EXCEPTION_IF(ptx_vector.empty(), error::wallet_internal_error, "No transactions"); + + // check every party in there does receive at least the required amount + std::unordered_map<account_public_address, std::pair<uint64_t, bool>> required; + for (const auto &d: dsts) + { + required[d.addr].first += d.amount; + required[d.addr].second = d.is_subaddress; + } + + // add change + uint64_t change = 0; + for (const auto &ptx: ptx_vector) + { + for (size_t idx: ptx.selected_transfers) + change += m_transfers[idx].amount(); + change -= ptx.fee; + } + for (const auto &r: required) + change -= r.second.first; + MDEBUG("Adding " << cryptonote::print_money(change) << " expected change"); + + for (const pending_tx &ptx: ptx_vector) + THROW_WALLET_EXCEPTION_IF(ptx.change_dts.addr != ptx_vector[0].change_dts.addr, error::wallet_internal_error, + "Change goes to several different addresses"); + const auto it = m_subaddresses.find(ptx_vector[0].change_dts.addr.m_spend_public_key); + THROW_WALLET_EXCEPTION_IF(it == m_subaddresses.end(), error::wallet_internal_error, "Change address is not ours"); + + required[ptx_vector[0].change_dts.addr].first += change; + required[ptx_vector[0].change_dts.addr].second = ptx_vector[0].change_dts.is_subaddress; + + for (const auto &r: required) + { + const account_public_address &address = r.first; + const crypto::public_key &view_pkey = address.m_view_public_key; + + uint64_t total_received = 0; + for (const auto &ptx: ptx_vector) + { + uint64_t received = 0; + try + { + std::string proof = get_tx_proof(ptx.tx, ptx.tx_key, ptx.additional_tx_keys, address, r.second.second, "automatic-sanity-check"); + check_tx_proof(ptx.tx, address, r.second.second, "automatic-sanity-check", proof, received); + } + catch (const std::exception &e) { received = 0; } + total_received += received; + } + + std::stringstream ss; + ss << "Total received by " << cryptonote::get_account_address_as_str(m_nettype, r.second.second, address) << ": " + << cryptonote::print_money(total_received) << ", expected " << cryptonote::print_money(r.second.first); + MDEBUG(ss.str()); + THROW_WALLET_EXCEPTION_IF(total_received < r.second.first, error::wallet_internal_error, ss.str()); + } + + return true; +} + std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; @@ -9635,6 +9733,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton ptx_vector.push_back(tx.ptx); } + uint64_t a = 0; + for (size_t idx: unused_transfers_indices) a += m_transfers[idx].amount(); + for (size_t idx: unused_dust_indices) a += m_transfers[idx].amount(); + std::vector<cryptonote::tx_destination_entry> synthetic_dsts(1, cryptonote::tx_destination_entry("", a, address, is_subaddress)); + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, synthetic_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + // if we made it this far, we're OK to actually send the transactions return ptx_vector; } @@ -9670,6 +9774,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_ hw::wallet_shim wallet_shim; setup_shim(&wallet_shim, this); aux_data.tx_recipients = dsts_info; + aux_data.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); tx_device_aux = aux_data.tx_device_aux; @@ -9886,7 +9991,7 @@ void wallet2::discard_unmixable_outputs() } } -bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const +bool wallet2::get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const { additional_tx_keys.clear(); const std::unordered_map<crypto::hash, crypto::secret_key>::const_iterator i = m_tx_keys.find(txid); @@ -9899,6 +10004,82 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s return true; } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) +{ + bool r = get_tx_key_cached(txid, tx_key, additional_tx_keys); + if (r) + { + return true; + } + + auto & hwdev = get_account().get_device(); + + // So far only Cold protocol devices are supported. + if (hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + { + return false; + } + + const auto tx_data_it = m_tx_device.find(txid); + if (tx_data_it == m_tx_device.end()) + { + MDEBUG("Aux data not found for txid: " << txid); + return false; + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + if (!dev_cold->is_get_tx_key_supported()) + { + MDEBUG("get_tx_key not supported by the device"); + return false; + } + + hw::device_cold::tx_key_data_t tx_key_data; + dev_cold->load_tx_key_data(tx_key_data, tx_data_it->second); + + // Load missing tx prefix hash + if (tx_key_data.tx_prefix_hash.empty()) + { + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash{}; + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash{}; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, + "Failed to get the right transaction from daemon"); + + tx_key_data.tx_prefix_hash = std::string(tx_prefix_hash.data, 32); + } + + std::vector<crypto::secret_key> tx_keys; + dev_cold->get_tx_key(tx_keys, tx_key_data, m_account.get_keys().m_view_secret_key); + if (tx_keys.empty()) + { + return false; + } + + tx_key = tx_keys[0]; + tx_keys.erase(tx_keys.begin()); + additional_tx_keys = tx_keys; + + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys) { // fetch tx from daemon and check if secret keys agree with corresponding public keys @@ -10190,41 +10371,8 @@ void wallet2::check_tx_key(const crypto::hash &txid, const crypto::secret_key &t check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); } -void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) +void wallet2::check_tx_key_helper(const cryptonote::transaction &tx, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received) const { - COMMAND_RPC_GET_TRANSACTIONS::request req; - COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); - req.decode_as_json = false; - req.prune = true; - m_daemon_rpc_mutex.lock(); - bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), - error::wallet_internal_error, "Failed to get transaction from daemon"); - - cryptonote::transaction tx; - crypto::hash tx_hash; - if (res.txs.size() == 1) - { - ok = get_pruned_tx(res.txs.front(), tx, tx_hash); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - } - else - { - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - tx_hash = cryptonote::get_transaction_hash(tx); - } - - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, - "Failed to get the right transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, - "The size of additional derivations is wrong"); - received = 0; hw::device &hwdev = m_account.get_device(); for (size_t n = 0; n < tx.vout.size(); ++n) @@ -10272,6 +10420,44 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de received += amount; } } +} + +void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) +{ + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash; + if (res.txs.size() == 1) + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } + else + { + cryptonote::blobdata tx_data; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } + + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, + "Failed to get the right transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, + "The size of additional derivations is wrong"); + + check_tx_key_helper(tx, derivation, additional_derivations, address, received); in_pool = res.txs.front().in_pool; confirmations = (uint64_t)-1; @@ -10286,9 +10472,55 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) { + // fetch tx pubkey from the daemon + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash; + if (res.txs.size() == 1) + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } + else + { + cryptonote::blobdata tx_data; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } + + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); + + // determine if the address is found in the subaddress hash table (i.e. whether the proof is outbound or inbound) + crypto::secret_key tx_key = crypto::null_skey; + std::vector<crypto::secret_key> additional_tx_keys; + const bool is_out = m_subaddresses.count(address.m_spend_public_key) == 0; + if (is_out) + { + THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file."); + } + + return get_tx_proof(tx, tx_key, additional_tx_keys, address, is_subaddress, message); +} + +std::string wallet2::get_tx_proof(const cryptonote::transaction &tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) const +{ // determine if the address is found in the subaddress hash table (i.e. whether the proof is outbound or inbound) const bool is_out = m_subaddresses.count(address.m_spend_public_key) == 0; + const crypto::hash txid = cryptonote::get_transaction_hash(tx); std::string prefix_data((const char*)&txid, sizeof(crypto::hash)); prefix_data += message; crypto::hash prefix_hash; @@ -10299,10 +10531,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac std::string sig_str; if (is_out) { - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file."); - const size_t num_sigs = 1 + additional_tx_keys.size(); shared_secret.resize(num_sigs); sig.resize(num_sigs); @@ -10337,37 +10565,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac } else { - // fetch tx pubkey from the daemon - COMMAND_RPC_GET_TRANSACTIONS::request req; - COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); - req.decode_as_json = false; - req.prune = true; - m_daemon_rpc_mutex.lock(); - bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), - error::wallet_internal_error, "Failed to get transaction from daemon"); - - cryptonote::transaction tx; - crypto::hash tx_hash; - if (res.txs.size() == 1) - { - ok = get_pruned_tx(res.txs.front(), tx, tx_hash); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - } - else - { - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - tx_hash = cryptonote::get_transaction_hash(tx); - } - - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); - crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); THROW_WALLET_EXCEPTION_IF(tx_pub_key == null_pkey, error::wallet_internal_error, "Tx pubkey was not found"); @@ -10409,9 +10606,7 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac for (size_t i = 1; i < num_sigs; ++i) THROW_WALLET_EXCEPTION_IF(!crypto::generate_key_derivation(shared_secret[i], rct::rct2sk(rct::I), additional_derivations[i - 1]), error::wallet_internal_error, "Failed to generate key derivation"); uint64_t received; - bool in_pool; - uint64_t confirmations; - check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); + check_tx_key_helper(tx, derivation, additional_derivations, address, received); THROW_WALLET_EXCEPTION_IF(!received, error::wallet_internal_error, tr("No funds received in this tx.")); // concatenate all signature strings @@ -10424,37 +10619,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received, bool &in_pool, uint64_t &confirmations) { - const bool is_out = sig_str.substr(0, 3) == "Out"; - const std::string header = is_out ? "OutProofV1" : "InProofV1"; - const size_t header_len = header.size(); - THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, - "Signature header check error"); - - // decode base58 - std::vector<crypto::public_key> shared_secret(1); - std::vector<crypto::signature> sig(1); - const size_t pk_len = tools::base58::encode(std::string((const char *)&shared_secret[0], sizeof(crypto::public_key))).size(); - const size_t sig_len = tools::base58::encode(std::string((const char *)&sig[0], sizeof(crypto::signature))).size(); - const size_t num_sigs = (sig_str.size() - header_len) / (pk_len + sig_len); - THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * (pk_len + sig_len), error::wallet_internal_error, - "Wrong signature size"); - shared_secret.resize(num_sigs); - sig.resize(num_sigs); - for (size_t i = 0; i < num_sigs; ++i) - { - std::string pk_decoded; - std::string sig_decoded; - const size_t offset = header_len + i * (pk_len + sig_len); - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset, pk_len), pk_decoded), error::wallet_internal_error, - "Signature decoding error"); - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset + pk_len, sig_len), sig_decoded), error::wallet_internal_error, - "Signature decoding error"); - THROW_WALLET_EXCEPTION_IF(sizeof(crypto::public_key) != pk_decoded.size() || sizeof(crypto::signature) != sig_decoded.size(), error::wallet_internal_error, - "Signature decoding error"); - memcpy(&shared_secret[i], pk_decoded.data(), sizeof(crypto::public_key)); - memcpy(&sig[i], sig_decoded.data(), sizeof(crypto::signature)); - } - // fetch tx pubkey from the daemon COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; @@ -10486,12 +10650,62 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); + if (!check_tx_proof(tx, address, is_subaddress, message, sig_str, received)) + return false; + + in_pool = res.txs.front().in_pool; + confirmations = (uint64_t)-1; + if (!in_pool) + { + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (err.empty()) + confirmations = bc_height - (res.txs.front().block_height + 1); + } + + return true; +} + +bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const +{ + const bool is_out = sig_str.substr(0, 3) == "Out"; + const std::string header = is_out ? "OutProofV1" : "InProofV1"; + const size_t header_len = header.size(); + THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, + "Signature header check error"); + + // decode base58 + std::vector<crypto::public_key> shared_secret(1); + std::vector<crypto::signature> sig(1); + const size_t pk_len = tools::base58::encode(std::string((const char *)&shared_secret[0], sizeof(crypto::public_key))).size(); + const size_t sig_len = tools::base58::encode(std::string((const char *)&sig[0], sizeof(crypto::signature))).size(); + const size_t num_sigs = (sig_str.size() - header_len) / (pk_len + sig_len); + THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * (pk_len + sig_len), error::wallet_internal_error, + "Wrong signature size"); + shared_secret.resize(num_sigs); + sig.resize(num_sigs); + for (size_t i = 0; i < num_sigs; ++i) + { + std::string pk_decoded; + std::string sig_decoded; + const size_t offset = header_len + i * (pk_len + sig_len); + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset, pk_len), pk_decoded), error::wallet_internal_error, + "Signature decoding error"); + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset + pk_len, sig_len), sig_decoded), error::wallet_internal_error, + "Signature decoding error"); + THROW_WALLET_EXCEPTION_IF(sizeof(crypto::public_key) != pk_decoded.size() || sizeof(crypto::signature) != sig_decoded.size(), error::wallet_internal_error, + "Signature decoding error"); + memcpy(&shared_secret[i], pk_decoded.data(), sizeof(crypto::public_key)); + memcpy(&sig[i], sig_decoded.data(), sizeof(crypto::signature)); + } + crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); THROW_WALLET_EXCEPTION_IF(tx_pub_key == null_pkey, error::wallet_internal_error, "Tx pubkey was not found"); std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); THROW_WALLET_EXCEPTION_IF(additional_tx_pub_keys.size() + 1 != num_sigs, error::wallet_internal_error, "Signature size mismatch with additional tx pubkeys"); + const crypto::hash txid = cryptonote::get_transaction_hash(tx); std::string prefix_data((const char*)&txid, sizeof(crypto::hash)); prefix_data += message; crypto::hash prefix_hash; @@ -10538,7 +10752,7 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account if (good_signature[i]) THROW_WALLET_EXCEPTION_IF(!crypto::generate_key_derivation(shared_secret[i], rct::rct2sk(rct::I), additional_derivations[i - 1]), error::wallet_internal_error, "Failed to generate key derivation"); - check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); + check_tx_key_helper(tx, derivation, additional_derivations, address, received); return true; } return false; @@ -11040,15 +11254,6 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle size_t pk_index = 0; hw::device &hwdev = m_account.get_device(); - const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); - std::vector<crypto::key_derivation> additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) - { - additional_derivations.push_back({}); - bool r = hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back()); - THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key derivation"); - } - while (find_tx_extra_field_by_type(tx_extra_fields, pub_key_field, pk_index++)) { const crypto::public_key tx_pub_key = pub_key_field.pub_key; crypto::key_derivation derivation; @@ -11058,16 +11263,15 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle for (size_t i = 0; i < td.m_tx.vout.size(); ++i) { tx_scan_info_t tx_scan_info; - check_acc_out_precomp(td.m_tx.vout[i], derivation, additional_derivations, i, tx_scan_info); + check_acc_out_precomp(td.m_tx.vout[i], derivation, {}, i, tx_scan_info); if (!tx_scan_info.error && tx_scan_info.received) return tx_pub_key; } } - // we found no key yielding an output - THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, - "Public key yielding at least one output wasn't found in the transaction extra"); - return crypto::null_pkey; + // we found no key yielding an output, but it might be in the additional + // tx pub keys only, which we do not need to check, so return the first one + return tx_pub_key; } bool wallet2::export_key_images(const std::string &filename) const @@ -11309,6 +11513,17 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag } PERF_TIMER_STOP(import_key_images_C); + // accumulate outputs before the updated data + for(size_t i = 0; i < offset; ++i) + { + const transfer_details &td = m_transfers[i]; + uint64_t amount = td.amount(); + if (td.m_spent) + spent += amount; + else + unspent += amount; + } + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { @@ -11472,32 +11687,52 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag PERF_TIMER_STOP(import_key_images_G); } - return m_transfers[signed_key_images.size() - 1].m_block_height; + // this can be 0 if we do not know the height + return m_transfers[signed_key_images.size() + offset - 1].m_block_height; } -bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) +bool wallet2::import_key_images(std::vector<crypto::key_image> key_images, size_t offset, boost::optional<std::unordered_set<size_t>> selected_transfers) { - if (key_images.size() > m_transfers.size()) + if (key_images.size() + offset > m_transfers.size()) { LOG_PRINT_L1("More key images returned that we know outputs for"); return false; } - for (size_t i = 0; i < key_images.size(); ++i) + for (size_t ki_idx = 0; ki_idx < key_images.size(); ++ki_idx) { - transfer_details &td = m_transfers[i]; - if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i]) - LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); - td.m_key_image = key_images[i]; - m_key_images[m_transfers[i].m_key_image] = i; + const size_t transfer_idx = ki_idx + offset; + if (selected_transfers && selected_transfers.get().find(transfer_idx) == selected_transfers.get().end()) + continue; + + transfer_details &td = m_transfers[transfer_idx]; + if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[ki_idx]) + LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << ki_idx << ": trusting imported one"); + td.m_key_image = key_images[ki_idx]; + m_key_images[td.m_key_image] = transfer_idx; td.m_key_image_known = true; td.m_key_image_request = false; td.m_key_image_partial = false; - m_pub_keys[m_transfers[i].get_public_key()] = i; + m_pub_keys[td.get_public_key()] = transfer_idx; } return true; } +bool wallet2::import_key_images(signed_tx_set & signed_tx, size_t offset, bool only_selected_transfers) +{ + std::unordered_set<size_t> selected_transfers; + if (only_selected_transfers) + { + for (const pending_tx & ptx : signed_tx.ptx) + { + for (const size_t s: ptx.selected_transfers) + selected_transfers.insert(s); + } + } + + return import_key_images(signed_tx.key_images, offset, only_selected_transfers ? boost::make_optional(selected_transfers) : boost::none); +} + wallet2::payment_container wallet2::export_payments() const { payment_container payments; @@ -12478,22 +12713,30 @@ wallet_device_callback * wallet2::get_device_callback() } return m_device_callback.get(); }//---------------------------------------------------------------------------------------------------- -void wallet2::on_button_request() +void wallet2::on_device_button_request(uint64_t code) { - if (0 != m_callback) - m_callback->on_button_request(); + if (nullptr != m_callback) + m_callback->on_device_button_request(code); } //---------------------------------------------------------------------------------------------------- -void wallet2::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> wallet2::on_device_pin_request() { - if (0 != m_callback) - m_callback->on_pin_request(pin); + if (nullptr != m_callback) + return m_callback->on_device_pin_request(); + return boost::none; } //---------------------------------------------------------------------------------------------------- -void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device) { - if (0 != m_callback) - m_callback->on_passphrase_request(on_device, passphrase); + if (nullptr != m_callback) + return m_callback->on_device_passphrase_request(on_device); + return boost::none; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_device_progress(const hw::device_progress& event) +{ + if (nullptr != m_callback) + m_callback->on_device_progress(event); } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_rpc_status(const std::string &s) const diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 1911a8e96..c25450fb3 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -103,9 +103,10 @@ namespace tools virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {} // Device callbacks - virtual void on_button_request() {} - virtual void on_pin_request(epee::wipeable_string & pin) {} - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual void on_device_button_request(uint64_t code) {} + virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; } + virtual void on_device_progress(const hw::device_progress& event) {}; // Common callbacks virtual void on_pool_tx_removed(const crypto::hash &txid) {} virtual ~i_wallet2_callback() {} @@ -115,9 +116,10 @@ namespace tools { public: wallet_device_callback(wallet2 * wallet): wallet(wallet) {}; - void on_button_request() override; - void on_pin_request(epee::wipeable_string & pin) override; - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override; + void on_button_request(uint64_t code=0) override; + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; + void on_progress(const hw::device_progress& event) override; private: wallet2 * wallet; }; @@ -797,6 +799,7 @@ namespace tools std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const; void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); @@ -1005,12 +1008,16 @@ namespace tools const std::string & device_derivation_path() const { return m_device_derivation_path; } void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } - bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; + bool get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); + bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys); void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); + void check_tx_key_helper(const cryptonote::transaction &tx, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received) const; std::string get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message); + std::string get_tx_proof(const cryptonote::transaction &tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) const; bool check_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received, bool &in_pool, uint64_t &confirmations); + bool check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const; std::string get_spend_proof(const crypto::hash &txid, const std::string &message); bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str); @@ -1128,7 +1135,8 @@ namespace tools std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); - bool import_key_images(std::vector<crypto::key_image> key_images); + bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); + bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; void update_pool_state(bool refreshed = false); @@ -1338,9 +1346,10 @@ namespace tools void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); wallet_device_callback * get_device_callback(); - void on_button_request(); - void on_pin_request(epee::wipeable_string & pin); - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + void on_device_button_request(uint64_t code); + boost::optional<epee::wipeable_string> on_device_pin_request(); + boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); + void on_device_progress(const hw::device_progress& event); std::string get_rpc_status(const std::string &s) const; void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const; diff --git a/tests/block_weight/CMakeLists.txt b/tests/block_weight/CMakeLists.txt index b0d716ea0..a59c9c1fc 100644 --- a/tests/block_weight/CMakeLists.txt +++ b/tests/block_weight/CMakeLists.txt @@ -42,4 +42,4 @@ target_link_libraries(block_weight add_test( NAME block_weight - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight) + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight) diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp index 57fcb497e..1e8974b78 100644 --- a/tests/block_weight/block_weight.cpp +++ b/tests/block_weight/block_weight.cpp @@ -72,6 +72,18 @@ public: virtual uint64_t height() const override { return blocks.size(); } virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight); + return ret; + } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight); + return ret; + } virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override { uint64_t h = height(); crypto::hash top = crypto::null_hash; diff --git a/tests/block_weight/block_weight.py b/tests/block_weight/block_weight.py index 06aaabb02..d6fd494e3 100755 --- a/tests/block_weight/block_weight.py +++ b/tests/block_weight/block_weight.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Simulate a maximal block attack on the Monero network # This uses the scheme proposed by ArticMine # Written by Sarang Nother diff --git a/tests/block_weight/compare.py b/tests/block_weight/compare.py index c6be05206..e058a7079 100755 --- a/tests/block_weight/compare.py +++ b/tests/block_weight/compare.py @@ -3,10 +3,17 @@ import sys import subprocess -print 'running: ', sys.argv[1] -S0 = subprocess.check_output(sys.argv[1], stderr=subprocess.STDOUT) -print 'running: ', sys.argv[2] -S1 = subprocess.check_output(sys.argv[2], stderr=subprocess.STDOUT) +if len(sys.argv) == 4: + first = [sys.argv[1], sys.argv[2]] + second = [sys.argv[3]] +else: + first = [sys.argv[1]] + second = [sys.argv[2]] + +print 'running: ', first +S0 = subprocess.check_output(first, stderr=subprocess.STDOUT) +print 'running: ', second +S1 = subprocess.check_output(second, stderr=subprocess.STDOUT) print 'comparing' if S0 != S1: sys.exit(1) diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 8bf5a9b08..f2bcb7787 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -754,7 +754,7 @@ struct get_test_options { }; //-------------------------------------------------------------------------- template<class t_test_class> -inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core **core) +inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core *core) { boost::program_options::options_description desc("Allowed options"); cryptonote::core::init_options(desc); @@ -768,8 +768,7 @@ inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cry if (!r) return false; - *core = new cryptonote::core(nullptr); - auto & c = **core; + auto & c = *core; // FIXME: make sure that vm has arg_testnet_on set to true or false if // this test needs for it to be so. @@ -825,9 +824,9 @@ inline bool replay_events_through_core_validate(std::vector<test_event_entry>& e template<class t_test_class> inline bool do_replay_events(std::vector<test_event_entry>& events) { - cryptonote::core * core; + cryptonote::core core(nullptr); bool ret = do_replay_events_get_core<t_test_class>(events, &core); - core->deinit(); + core.deinit(); return ret; } //-------------------------------------------------------------------------- diff --git a/tests/core_tests/wallet_tools.cpp b/tests/core_tests/wallet_tools.cpp index ff7ce3a34..616774d18 100644 --- a/tests/core_tests/wallet_tools.cpp +++ b/tests/core_tests/wallet_tools.cpp @@ -17,7 +17,6 @@ void wallet_accessor_test::set_account(tools::wallet2 * wallet, cryptonote::acco { wallet->clear(); wallet->m_account = account; - wallet->m_nettype = MAINNET; wallet->m_key_device_type = account.get_device().get_type(); wallet->m_account_public_address = account.get_keys().m_account_address; diff --git a/tests/gtest/include/gtest/internal/gtest-port-arch.h b/tests/gtest/include/gtest/internal/gtest-port-arch.h index 74ab94905..83022c82c 100644 --- a/tests/gtest/include/gtest/internal/gtest-port-arch.h +++ b/tests/gtest/include/gtest/internal/gtest-port-arch.h @@ -88,6 +88,8 @@ # define GTEST_OS_OPENBSD 1 #elif defined __QNX__ # define GTEST_OS_QNX 1 +#elif defined(__NetBSD__) +# define GTEST_OS_NETBSD 1 #endif // __CYGWIN__ #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ diff --git a/tests/gtest/include/gtest/internal/gtest-port.h b/tests/gtest/include/gtest/internal/gtest-port.h index cb0639b4b..2896dfe64 100644 --- a/tests/gtest/include/gtest/internal/gtest-port.h +++ b/tests/gtest/include/gtest/internal/gtest-port.h @@ -790,7 +790,8 @@ using ::std::tuple_size; (GTEST_OS_MAC && !GTEST_OS_IOS) || \ (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \ - GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD) + GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD || \ + GTEST_OS_NETBSD) # define GTEST_HAS_DEATH_TEST 1 #endif diff --git a/tests/trezor/CMakeLists.txt b/tests/trezor/CMakeLists.txt index 67c2f8438..15ed5668d 100644 --- a/tests/trezor/CMakeLists.txt +++ b/tests/trezor/CMakeLists.txt @@ -27,11 +27,15 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set(trezor_tests_sources + tools.cpp + daemon.cpp trezor_tests.cpp ../core_tests/chaingen.cpp ../core_tests/wallet_tools.cpp) set(trezor_tests_headers + tools.h + daemon.h trezor_tests.h ../core_tests/chaingen.h ../core_tests/wallet_tools.h) @@ -50,6 +54,15 @@ target_link_libraries(trezor_tests device device_trezor wallet + wallet_api + rpc + cryptonote_protocol + daemon_rpc_server + ${Boost_CHRONO_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${ZMQ_LIB} ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) diff --git a/tests/trezor/daemon.cpp b/tests/trezor/daemon.cpp new file mode 100644 index 000000000..5e987793a --- /dev/null +++ b/tests/trezor/daemon.cpp @@ -0,0 +1,368 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "daemon.h" +#include <common/command_line.h> + +using namespace std; +using namespace daemonize; +namespace po = boost::program_options; + +bool mock_rpc_daemon::on_send_raw_tx_2(const cryptonote::COMMAND_RPC_SEND_RAW_TX::request& req, cryptonote::COMMAND_RPC_SEND_RAW_TX::response& res, const cryptonote::core_rpc_server::connection_context *ctx) +{ + cryptonote::COMMAND_RPC_SEND_RAW_TX::request req2(req); + req2.do_not_relay = true; // Do not relay in test setup, only one daemon running. + return cryptonote::core_rpc_server::on_send_raw_tx(req2, res, ctx); +} + +void mock_daemon::init_options(boost::program_options::options_description & option_spec) +{ + cryptonote::core::init_options(option_spec); + t_node_server::init_options(option_spec); + cryptonote::core_rpc_server::init_options(option_spec); + + command_line::add_arg(option_spec, daemon_args::arg_zmq_rpc_bind_ip); + command_line::add_arg(option_spec, daemon_args::arg_zmq_rpc_bind_port); +} + +void mock_daemon::default_options(boost::program_options::variables_map & vm) +{ + std::vector<std::string> exclusive_nodes{"127.0.0.1:65525"}; + tools::options::set_option(vm, nodetool::arg_p2p_add_exclusive_node, po::variable_value(exclusive_nodes, false)); + + tools::options::set_option(vm, nodetool::arg_p2p_bind_ip, po::variable_value(std::string("127.0.0.1"), false)); + tools::options::set_option(vm, nodetool::arg_no_igd, po::variable_value(true, false)); + tools::options::set_option(vm, cryptonote::arg_offline, po::variable_value(true, false)); + tools::options::set_option(vm, "disable-dns-checkpoints", po::variable_value(true, false)); + + const char *test_mainnet = getenv("TEST_MAINNET"); + if (!test_mainnet || atoi(test_mainnet) == 0) + { + tools::options::set_option(vm, cryptonote::arg_testnet_on, po::variable_value(true, false)); + } + + // By default pick non-standard ports to avoid confusion with possibly locally running daemons (mainnet/testnet) + const char *test_p2p_port = getenv("TEST_P2P_PORT"); + auto p2p_port = std::string(test_p2p_port && strlen(test_p2p_port) > 0 ? test_p2p_port : "61340"); + tools::options::set_option(vm, nodetool::arg_p2p_bind_port, po::variable_value(p2p_port, false)); + + const char *test_rpc_port = getenv("TEST_RPC_PORT"); + auto rpc_port = std::string(test_rpc_port && strlen(test_rpc_port) > 0 ? test_rpc_port : "61341"); + tools::options::set_option(vm, cryptonote::core_rpc_server::arg_rpc_bind_port, po::variable_value(rpc_port, false)); + + const char *test_zmq_port = getenv("TEST_ZMQ_PORT"); + auto zmq_port = std::string(test_zmq_port && strlen(test_zmq_port) > 0 ? test_zmq_port : "61342"); + tools::options::set_option(vm, daemon_args::arg_zmq_rpc_bind_port, po::variable_value(zmq_port, false)); + + po::notify(vm); +} + +void mock_daemon::set_ports(boost::program_options::variables_map & vm, unsigned initial_port) +{ + CHECK_AND_ASSERT_THROW_MES(initial_port < 65535-2, "Invalid port number"); + tools::options::set_option(vm, nodetool::arg_p2p_bind_port, po::variable_value(std::to_string(initial_port), false)); + tools::options::set_option(vm, cryptonote::core_rpc_server::arg_rpc_bind_port, po::variable_value(std::to_string(initial_port + 1), false)); + tools::options::set_option(vm, daemon_args::arg_zmq_rpc_bind_port, po::variable_value(std::to_string(initial_port + 2), false)); + po::notify(vm); +} + +void mock_daemon::load_params(boost::program_options::variables_map const & vm) +{ + m_p2p_bind_port = command_line::get_arg(vm, nodetool::arg_p2p_bind_port); + m_rpc_bind_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + m_zmq_bind_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port); + m_network_type = command_line::get_arg(vm, cryptonote::arg_testnet_on) ? cryptonote::TESTNET : cryptonote::MAINNET; +} + +mock_daemon::~mock_daemon() +{ + if (!m_terminated) + { + try + { + stop(); + } + catch (...) + { + MERROR("Failed to stop"); + } + } + + if (!m_deinitalized) + { + deinit(); + } +} + +void mock_daemon::init() +{ + m_deinitalized = false; + const auto main_rpc_port = command_line::get_arg(m_vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + m_rpc_server.nettype(m_network_type); + + CHECK_AND_ASSERT_THROW_MES(m_protocol.init(m_vm), "Failed to initialize cryptonote protocol."); + CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port), "Failed to initialize RPC server."); + + if (m_start_p2p) + CHECK_AND_ASSERT_THROW_MES(m_server.init(m_vm), "Failed to initialize p2p server."); + + if(m_http_client.is_connected()) + m_http_client.disconnect(); + + CHECK_AND_ASSERT_THROW_MES(m_http_client.set_server(rpc_addr(), boost::none), "RPC client init fail"); +} + +void mock_daemon::deinit() +{ + try + { + m_rpc_server.deinit(); + } + catch (...) + { + MERROR("Failed to deinitialize RPC server..."); + } + + if (m_start_p2p) + { + try + { + m_server.deinit(); + } + catch (...) + { + MERROR("Failed to deinitialize p2p..."); + } + } + + try + { + m_protocol.deinit(); + m_protocol.set_p2p_endpoint(nullptr); + } + catch (...) + { + MERROR("Failed to stop cryptonote protocol!"); + } + + m_deinitalized = true; +} + +void mock_daemon::init_and_run() +{ + init(); + run(); +} + +void mock_daemon::stop_and_deinit() +{ + stop(); + deinit(); +} + +void mock_daemon::try_init_and_run(boost::optional<unsigned> initial_port) +{ + const unsigned max_attempts = 3; + for(unsigned attempts=0; attempts < max_attempts; ++attempts) + { + if (initial_port) + { + set_ports(m_vm, initial_port.get()); + load_params(m_vm); + MDEBUG("Ports changed, RPC: " << rpc_addr()); + } + + try + { + init_and_run(); + return; + } + catch(const std::exception &e) + { + MWARNING("Could not init and start, attempt: " << attempts << ", reason: " << e.what()); + if (attempts + 1 >= max_attempts) + { + throw; + } + } + } +} + +void mock_daemon::run() +{ + m_run_thread = boost::thread(boost::bind(&mock_daemon::run_main, this)); +} + +bool mock_daemon::run_main() +{ + CHECK_AND_ASSERT_THROW_MES(!m_terminated, "Can't run stopped daemon"); + CHECK_AND_ASSERT_THROW_MES(!m_start_zmq || m_start_p2p, "ZMQ requires P2P"); + boost::thread stop_thread = boost::thread([this] { + while (!this->m_stopped) + epee::misc_utils::sleep_no_w(100); + this->stop_p2p(); + }); + + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + m_stopped = true; + stop_thread.join(); + }); + + try + { + CHECK_AND_ASSERT_THROW_MES(m_rpc_server.run(2, false), "Failed to start RPC"); + cryptonote::rpc::DaemonHandler rpc_daemon_handler(*m_core, m_server); + cryptonote::rpc::ZmqServer zmq_server(rpc_daemon_handler); + + if (m_start_zmq) + { + if (!zmq_server.addTCPSocket("127.0.0.1", m_zmq_bind_port)) + { + MERROR("Failed to add TCP Socket (127.0.0.1:" << m_zmq_bind_port << ") to ZMQ RPC Server"); + + stop_rpc(); + return false; + } + + MINFO("Starting ZMQ server..."); + zmq_server.run(); + + MINFO("ZMQ server started at 127.0.0.1: " << m_zmq_bind_port); + } + + if (m_start_p2p) + { + m_server.run(); // blocks until p2p goes down + } + else + { + while (!this->m_stopped) + epee::misc_utils::sleep_no_w(100); + } + + if (m_start_zmq) + zmq_server.stop(); + + stop_rpc(); + return true; + } + catch (std::exception const & ex) + { + MFATAL("Uncaught exception! " << ex.what()); + return false; + } + catch (...) + { + MFATAL("Uncaught exception!"); + return false; + } +} + +void mock_daemon::stop() +{ + CHECK_AND_ASSERT_THROW_MES(!m_terminated, "Can't stop stopped daemon"); + m_stopped = true; + m_terminated = true; + m_run_thread.join(); +} + +void mock_daemon::stop_rpc() +{ + m_rpc_server.send_stop_signal(); + m_rpc_server.timed_wait_server_stop(5000); +} + +void mock_daemon::stop_p2p() +{ + if (m_start_p2p) + m_server.send_stop_signal(); +} + +void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address) +{ + bool blocks_mined = false; + const uint64_t start_height = get_height(); + const auto mining_timeout = std::chrono::seconds(30); + MDEBUG("Current height before mining: " << start_height); + + start_mining(miner_address); + auto mining_started = std::chrono::system_clock::now(); + + while(true) { + epee::misc_utils::sleep_no_w(100); + const uint64_t cur_height = get_height(); + + if (cur_height - start_height >= num_blocks) + { + MDEBUG("Cur blocks: " << cur_height << " start: " << start_height); + blocks_mined = true; + break; + } + + auto current_time = std::chrono::system_clock::now(); + if (mining_timeout < current_time - mining_started) + { + break; + } + } + + stop_mining(); + CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit"); +} + +constexpr const std::chrono::seconds mock_daemon::rpc_timeout; + +void mock_daemon::start_mining(const std::string &miner_address, uint64_t threads_count, bool do_background_mining, bool ignore_battery) +{ + cryptonote::COMMAND_RPC_START_MINING::request req; + req.miner_address = miner_address; + req.threads_count = threads_count; + req.do_background_mining = do_background_mining; + req.ignore_battery = ignore_battery; + + cryptonote::COMMAND_RPC_START_MINING::response resp; + bool r = epee::net_utils::invoke_http_json("/start_mining", req, resp, m_http_client, rpc_timeout); + CHECK_AND_ASSERT_THROW_MES(r, "RPC error - start mining"); + CHECK_AND_ASSERT_THROW_MES(resp.status != CORE_RPC_STATUS_BUSY, "Daemon busy"); + CHECK_AND_ASSERT_THROW_MES(resp.status == CORE_RPC_STATUS_OK, "Daemon response invalid: " << resp.status); +} + +void mock_daemon::stop_mining() +{ + cryptonote::COMMAND_RPC_STOP_MINING::request req; + cryptonote::COMMAND_RPC_STOP_MINING::response resp; + bool r = epee::net_utils::invoke_http_json("/stop_mining", req, resp, m_http_client, rpc_timeout); + CHECK_AND_ASSERT_THROW_MES(r, "RPC error - stop mining"); + CHECK_AND_ASSERT_THROW_MES(resp.status != CORE_RPC_STATUS_BUSY, "Daemon busy"); + CHECK_AND_ASSERT_THROW_MES(resp.status == CORE_RPC_STATUS_OK, "Daemon response invalid: " << resp.status); +} + +uint64_t mock_daemon::get_height() +{ + return m_core->get_blockchain_storage().get_current_blockchain_height(); +} diff --git a/tests/trezor/daemon.h b/tests/trezor/daemon.h new file mode 100644 index 000000000..046b09a5d --- /dev/null +++ b/tests/trezor/daemon.h @@ -0,0 +1,154 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "misc_log_ex.h" +#include "daemon/daemon.h" +#include "rpc/daemon_handler.h" +#include "rpc/zmq_server.h" +#include "common/password.h" +#include "common/util.h" +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "daemon/protocol.h" +#include "daemon/rpc.h" +#include "daemon/command_server.h" +#include "daemon/command_server.h" +#include "daemon/command_line_args.h" +#include "version.h" +#include "tools.h" + + +class mock_rpc_daemon : public cryptonote::core_rpc_server { +public: + mock_rpc_daemon( + cryptonote::core& cr + , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p + ): cryptonote::core_rpc_server(cr, p2p) {} + + static void init_options(boost::program_options::options_description& desc){ cryptonote::core_rpc_server::init_options(desc); } + cryptonote::network_type nettype() const { return m_network_type; } + void nettype(cryptonote::network_type nettype) { m_network_type = nettype; } + + CHAIN_HTTP_TO_MAP2(cryptonote::core_rpc_server::connection_context); //forward http requests to uri map + BEGIN_URI_MAP2() + MAP_URI_AUTO_JON2("/send_raw_transaction", on_send_raw_tx_2, cryptonote::COMMAND_RPC_SEND_RAW_TX) + MAP_URI_AUTO_JON2("/sendrawtransaction", on_send_raw_tx_2, cryptonote::COMMAND_RPC_SEND_RAW_TX) + else { // Default to parent for non-overriden callbacks + return cryptonote::core_rpc_server::handle_http_request_map(query_info, response_info, m_conn_context); + } + END_URI_MAP2() + + bool on_send_raw_tx_2(const cryptonote::COMMAND_RPC_SEND_RAW_TX::request& req, cryptonote::COMMAND_RPC_SEND_RAW_TX::response& res, const cryptonote::core_rpc_server::connection_context *ctx); + +protected: + cryptonote::network_type m_network_type; +}; + +class mock_daemon { +public: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + typedef nodetool::node_server<t_protocol_raw> t_node_server; + + static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(60); + + cryptonote::core * m_core; + t_protocol_raw m_protocol; + mock_rpc_daemon m_rpc_server; + t_node_server m_server; + cryptonote::network_type m_network_type; + epee::net_utils::http::http_simple_client m_http_client; + + bool m_start_p2p; + bool m_start_zmq; + boost::program_options::variables_map m_vm; + + std::string m_p2p_bind_port; + std::string m_rpc_bind_port; + std::string m_zmq_bind_port; + + std::atomic<bool> m_stopped; + std::atomic<bool> m_terminated; + std::atomic<bool> m_deinitalized; + boost::thread m_run_thread; + + mock_daemon( + cryptonote::core * core, + boost::program_options::variables_map const & vm + ) + : m_core(core) + , m_vm(vm) + , m_start_p2p(false) + , m_start_zmq(false) + , m_terminated(false) + , m_deinitalized(false) + , m_stopped(false) + , m_protocol{*core, nullptr, command_line::get_arg(vm, cryptonote::arg_offline)} + , m_server{m_protocol} + , m_rpc_server{*core, m_server} + { + // Handle circular dependencies + m_protocol.set_p2p_endpoint(&m_server); + m_core->set_cryptonote_protocol(&m_protocol); + load_params(vm); + } + + virtual ~mock_daemon(); + + static void init_options(boost::program_options::options_description & option_spec); + static void default_options(boost::program_options::variables_map & vm); + static void set_ports(boost::program_options::variables_map & vm, unsigned initial_port); + + mock_daemon * set_start_p2p(bool fl) { m_start_p2p = fl; return this; } + mock_daemon * set_start_zmq(bool fl) { m_start_zmq = fl; return this; } + + void init(); + void deinit(); + void run(); + bool run_main(); + void stop(); + void stop_p2p(); + void stop_rpc(); + void init_and_run(); + void stop_and_deinit(); + void try_init_and_run(boost::optional<unsigned> initial_port=boost::none); + + void mine_blocks(size_t num_blocks, const std::string &miner_address); + void start_mining(const std::string &miner_address, uint64_t threads_count=1, bool do_background_mining=false, bool ignore_battery=true); + void stop_mining(); + uint64_t get_height(); + + void load_params(boost::program_options::variables_map const & vm); + + std::string zmq_addr() const { return std::string("127.0.0.1:") + m_zmq_bind_port; } + std::string rpc_addr() const { return std::string("127.0.0.1:") + m_rpc_bind_port; } + std::string p2p_addr() const { return std::string("127.0.0.1:") + m_p2p_bind_port; } + cryptonote::network_type nettype() const { return m_network_type; } + cryptonote::core * core() const { return m_core; } +}; diff --git a/tests/trezor/tools.cpp b/tests/trezor/tools.cpp new file mode 100644 index 000000000..432350cf6 --- /dev/null +++ b/tests/trezor/tools.cpp @@ -0,0 +1,56 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "tools.h" + +namespace tools { + +namespace po = boost::program_options; + +void options::set_option(boost::program_options::variables_map &vm, const std::string & key, const po::variable_value &pv) +{ + auto it = vm.find(key); + if (it == vm.end()) + { + vm.insert(std::make_pair(key, pv)); + } + else + { + it->second = pv; + } +} + +void options::build_options(boost::program_options::variables_map & vm, const po::options_description & desc_params) +{ + const char *argv[2] = {nullptr}; + po::store(po::parse_command_line(1, argv, desc_params), vm); + po::notify(vm); +} + +} + diff --git a/tests/trezor/tools.h b/tests/trezor/tools.h new file mode 100644 index 000000000..d348a5137 --- /dev/null +++ b/tests/trezor/tools.h @@ -0,0 +1,61 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "misc_log_ex.h" +#include "daemon/daemon.h" +#include "rpc/daemon_handler.h" +#include "rpc/zmq_server.h" +#include "common/password.h" +#include "common/util.h" +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "daemon/protocol.h" +#include "daemon/rpc.h" +#include "daemon/command_server.h" +#include "daemon/command_server.h" +#include "daemon/command_line_args.h" +#include "version.h" + +namespace tools { + +class options { +public: + static void set_option(boost::program_options::variables_map &vm, const std::string &key, const boost::program_options::variable_value &pv); + static void build_options(boost::program_options::variables_map & vm, const boost::program_options::options_description & desc_params); + + template<typename T, bool required, bool dependent, int NUM_DEPS> + static void set_option(boost::program_options::variables_map &vm, const command_line::arg_descriptor<T, required, dependent, NUM_DEPS> &arg, const boost::program_options::variable_value &pv) + { + set_option(vm, arg.name, pv); + } +}; + +}; + diff --git a/tests/trezor/trezor_tests.cpp b/tests/trezor/trezor_tests.cpp index c2b46f698..310fa45f1 100644 --- a/tests/trezor/trezor_tests.cpp +++ b/tests/trezor/trezor_tests.cpp @@ -41,6 +41,7 @@ using namespace cryptonote; #include "common/util.h" #include "common/command_line.h" #include "trezor_tests.h" +#include "tools.h" #include "device/device_cold.hpp" #include "device_trezor/device_trezor.hpp" @@ -57,7 +58,7 @@ namespace const command_line::arg_descriptor<bool> arg_fix_chain = {"fix_chain", "If chain_patch is given and file cannot be used, it is ignored and overwriten", false}; } - +#define HW_TREZOR_NAME "Trezor" #define TREZOR_ACCOUNT_ORDERING &m_miner_account, &m_alice_account, &m_bob_account, &m_eve_account #define TREZOR_COMMON_TEST_CASE(genclass, CORE, BASE) \ rollback_chain(CORE, BASE.head_block()); \ @@ -70,14 +71,17 @@ namespace #define TREZOR_SETUP_CHAIN(NAME) do { \ ++tests_count; \ try { \ - setup_chain(&core, trezor_base, chain_path, fix_chain); \ + setup_chain(core, trezor_base, chain_path, fix_chain, vm_core); \ } catch (const std::exception& ex) { \ failed_tests.emplace_back("gen_trezor_base " #NAME); \ } \ } while(0) + +static device_trezor_test *trezor_device = nullptr; +static device_trezor_test *ensure_trezor_test_device(); static void rollback_chain(cryptonote::core * core, const cryptonote::block & head); -static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain); +static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core); int main(int argc, char* argv[]) { @@ -123,38 +127,76 @@ int main(int argc, char* argv[]) const bool heavy_tests = command_line::get_arg(vm, arg_heavy_tests); const bool fix_chain = command_line::get_arg(vm, arg_fix_chain); - hw::trezor::register_all(); + hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device()); + // hw::trezor::register_all(); // We use our shim instead. // Bootstrapping common chain & accounts - cryptonote::core * core = nullptr; + const uint8_t initial_hf = 9; + const uint8_t max_hf = 10; + + cryptonote::core core_obj(nullptr); + cryptonote::core * const core = &core_obj; + std::shared_ptr<mock_daemon> daemon = nullptr; gen_trezor_base trezor_base; trezor_base.setup_args(trezor_path, heavy_tests); - trezor_base.rct_config({rct::RangeProofPaddedBulletproof, 1}); // HF9 tests + trezor_base.set_hard_fork(initial_hf); - TREZOR_SETUP_CHAIN("HF9"); - - // Individual test cases using shared pre-generated blockchain. - TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync, core, trezor_base); + // Arguments for core & daemon + po::variables_map vm_core; + po::options_description desc_params_core("Core"); + mock_daemon::init_options(desc_params_core); + tools::options::build_options(vm_core, desc_params_core); + mock_daemon::default_options(vm_core); // Transaction tests - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_2sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_1norm_2sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base); - TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base); + for(uint8_t hf=initial_hf; hf <= max_hf; ++hf) + { + MDEBUG("Transaction tests for HF " << (int)hf); + if (hf > initial_hf) + { + daemon->stop_and_deinit(); + daemon = nullptr; + trezor_base.daemon(nullptr); + } + + trezor_base.set_hard_fork(hf); + TREZOR_SETUP_CHAIN(std::string("HF") + std::to_string((int)hf)); + + daemon = std::make_shared<mock_daemon>(core, vm_core); + CHECK_AND_ASSERT_THROW_MES(daemon->nettype() == trezor_base.nettype(), "Serialized chain network type does not match"); + + daemon->try_init_and_run(); + trezor_base.daemon(daemon); + + // Hard-fork independent tests + if (hf == initial_hf) + { + TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_without_refresh, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_live_refresh, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_with_refresh, core, trezor_base); + } + + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_2sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_1norm_2sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base); + TREZOR_COMMON_TEST_CASE(wallet_api_tests, core, trezor_base); + } if (trezor_base.heavy_tests()) { TREZOR_COMMON_TEST_CASE(gen_trezor_many_utxo, core, trezor_base); } + daemon->stop(); core->deinit(); el::Level level = (failed_tests.empty() ? el::Level::Info : el::Level::Error); MLOG(level, "\nREPORT:"); @@ -243,7 +285,36 @@ static bool serialize_chain_to_file(std::vector<test_event_entry>& events, gen_t CATCH_ENTRY_L0("serialize_chain_to_file", false); } -static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain) +template<class t_test_class> +static bool init_core_replay_events(std::vector<test_event_entry>& events, cryptonote::core * core, const po::variables_map & vm_core) +{ + // this test needs for it to be so. + get_test_options<t_test_class> gto; + + // Hardforks can be specified in events. + v_hardforks_t hardforks; + cryptonote::test_options test_options_tmp{}; + const cryptonote::test_options * test_options_ = >o.test_options; + if (extract_hard_forks(events, hardforks)){ + hardforks.push_back(std::make_pair((uint8_t)0, (uint64_t)0)); // terminator + test_options_tmp.hard_forks = hardforks.data(); + test_options_ = &test_options_tmp; + } + + core->deinit(); + CHECK_AND_ASSERT_THROW_MES(core->init(vm_core, test_options_), "Core init failed"); + core->get_blockchain_storage().get_db().set_batch_transactions(true); + + // start with a clean pool + std::vector<crypto::hash> pool_txs; + CHECK_AND_ASSERT_THROW_MES(core->get_pool_transaction_hashes(pool_txs), "Failed to flush txpool"); + core->get_blockchain_storage().flush_txes_from_pool(pool_txs); + + t_test_class validator; + return replay_events_through_core<t_test_class>(*core, events, validator); +} + +static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core) { std::vector<test_event_entry> events; const bool do_serialize = !chain_path.empty(); @@ -272,6 +343,7 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, { try { + trezor_base.clear(); generated = trezor_base.generate(events); if (generated && !loaded && do_serialize) @@ -287,7 +359,7 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, } trezor_base.fix_hf(events); - if (generated && do_replay_events_get_core<gen_trezor_base>(events, core)) + if (generated && init_core_replay_events<gen_trezor_base>(events, core, vm_core)) { MGINFO_GREEN("#TEST-chain-init# Succeeded "); } @@ -298,6 +370,14 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, } } +static device_trezor_test *ensure_trezor_test_device(){ + if (!trezor_device) { + trezor_device = new device_trezor_test(); + trezor_device->set_name(HW_TREZOR_NAME); + } + return trezor_device; +} + static void add_hforks(std::vector<test_event_entry>& events, const v_hardforks_t& hard_forks) { event_replay_settings repl_set; @@ -483,8 +563,20 @@ static std::vector<tools::wallet2*> vct_wallets(tools::wallet2* w1=nullptr, tool return res; } +static uint64_t get_available_funds(tools::wallet2* wallet, uint32_t account=0) +{ + tools::wallet2::transfer_container transfers; + wallet->get_transfers(transfers); + uint64_t sum = 0; + for(const auto & cur : transfers) + { + sum += !cur.m_spent && cur.m_subaddr_index.major == account ? cur.amount() : 0; + } + return sum; +} + // gen_trezor_base -const uint64_t gen_trezor_base::m_ts_start = 1338224400; +const uint64_t gen_trezor_base::m_ts_start = 1397862000; // As default wallet timestamp is 1397516400 const uint64_t gen_trezor_base::m_wallet_ts = m_ts_start - 60*60*24*4; const std::string gen_trezor_base::m_device_name = "Trezor:udp"; const std::string gen_trezor_base::m_master_seed_str = "14821d0bc5659b24cafbc889dc4fc60785ee08b65d71c525f81eeaba4f3a570f"; @@ -494,12 +586,16 @@ const std::string gen_trezor_base::m_alice_view_private = "a6ccd4ac344a295d1387f gen_trezor_base::gen_trezor_base(){ m_rct_config = {rct::RangeProofPaddedBulletproof, 1}; + m_test_get_tx_key = true; + m_network_type = cryptonote::TESTNET; } gen_trezor_base::gen_trezor_base(const gen_trezor_base &other): m_generator(other.m_generator), m_bt(other.m_bt), m_miner_account(other.m_miner_account), m_bob_account(other.m_bob_account), m_alice_account(other.m_alice_account), m_eve_account(other.m_eve_account), - m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config) + m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config), + m_heavy_tests(other.m_heavy_tests), m_test_get_tx_key(other.m_test_get_tx_key), m_live_refresh_enabled(other.m_live_refresh_enabled), + m_network_type(other.m_network_type), m_daemon(other.m_daemon) { } @@ -513,33 +609,27 @@ void gen_trezor_base::setup_args(const std::string & trezor_path, bool heavy_tes void gen_trezor_base::setup_trezor() { hw::device &hwdev = hw::get_device(m_trezor_path); - m_trezor = dynamic_cast<hw::trezor::device_trezor *>(&hwdev); - CHECK_AND_ASSERT_THROW_MES(m_trezor, "Dynamic cast failed"); - - m_trezor->set_debug(true); // debugging commands on Trezor (auto-confirm transactions) - - CHECK_AND_ASSERT_THROW_MES(m_trezor->set_name(m_trezor_path), "Could not set device name " << m_trezor_path); - m_trezor->set_network_type(MAINNET); - m_trezor->set_derivation_path(""); // empty derivation path + auto trezor = dynamic_cast<device_trezor_test *>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(trezor, "Dynamic cast failed"); - CHECK_AND_ASSERT_THROW_MES(m_trezor->init(), "Could not initialize the device " << m_trezor_path); - CHECK_AND_ASSERT_THROW_MES(m_trezor->connect(), "Could not connect to the device " << m_trezor_path); - m_trezor->wipe_device(); - m_trezor->load_device(m_device_seed); - m_trezor->release(); - m_trezor->disconnect(); + trezor->setup_for_tests(m_trezor_path, m_device_seed, m_network_type); + m_trezor = trezor; } void gen_trezor_base::fork(gen_trezor_base & other) { other.m_generator = m_generator; other.m_bt = m_bt; + other.m_network_type = m_network_type; + other.m_daemon = m_daemon; other.m_events = m_events; other.m_head = m_head; other.m_hard_forks = m_hard_forks; other.m_trezor_path = m_trezor_path; other.m_heavy_tests = m_heavy_tests; other.m_rct_config = m_rct_config; + other.m_test_get_tx_key = m_test_get_tx_key; + other.m_live_refresh_enabled = m_live_refresh_enabled; other.m_miner_account = m_miner_account; other.m_bob_account = m_bob_account; @@ -577,10 +667,22 @@ void gen_trezor_base::init_fields() m_alice_account.set_createtime(m_wallet_ts); } +void gen_trezor_base::update_client_settings() +{ + auto dev_trezor = dynamic_cast<::hw::trezor::device_trezor*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_trezor, "Could not cast to device_trezor"); + + dev_trezor->set_live_refresh_enabled(m_live_refresh_enabled); +} + bool gen_trezor_base::generate(std::vector<test_event_entry>& events) { init_fields(); setup_trezor(); + + m_live_refresh_enabled = false; + update_client_settings(); + m_alice_account.create_from_device(*m_trezor); m_alice_account.set_createtime(m_wallet_ts); @@ -589,7 +691,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) cryptonote::block blk_gen; std::vector<size_t> block_weights; - generate_genesis_block(blk_gen, get_config(MAINNET).GENESIS_TX, get_config(MAINNET).GENESIS_NONCE); + generate_genesis_block(blk_gen, get_config(m_network_type).GENESIS_TX, get_config(m_network_type).GENESIS_NONCE); events.push_back(blk_gen); generator.add_block(blk_gen, 0, block_weights, 0); @@ -644,8 +746,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) MDEBUG("Hardfork height: " << hardfork_height << " at block: " << get_block_hash(blk_4r)); // RCT transactions, wallets have to be used, wallet init - m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true)); + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); @@ -659,7 +761,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) auto addr_alice_sub_1_2 = m_wl_alice->get_subaddress({1, 2}); // Miner -> Bob, RCT funds - MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(5), 10, blk_4); + MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(50), 10, blk_4); const size_t target_rct = m_heavy_tests ? 105 : 15; for(size_t i = 0; i < target_rct; ++i) @@ -688,9 +790,9 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) // Simple RCT transactions MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(7), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(1), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(3), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(4), 10, blk_4); + MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(10), 10, blk_4); + MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(30), 10, blk_4); + MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(40), 10, blk_4); MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_5, blk_4r, m_miner_account, txs_blk_5, CUR_HF); // Simple transaction check @@ -704,6 +806,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) // RCT transactions, wallets have to be used wallet_tools::process_transactions(m_wl_alice.get(), events, blk_5r, m_bt); wallet_tools::process_transactions(m_wl_bob.get(), events, blk_5r, m_bt); + MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get())); + MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get())); // Send Alice -> Bob, manually constructed. Simple TX test, precondition. cryptonote::transaction tx_1; @@ -768,18 +872,21 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events) m_eve_account.set_createtime(m_wallet_ts); setup_trezor(); + update_client_settings(); m_alice_account.create_from_device(*m_trezor); m_alice_account.set_createtime(m_wallet_ts); - m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_eve.reset(new tools::wallet2(MAINNET, 1, true)); + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); wallet_tools::process_transactions(m_wl_alice.get(), events, m_head, m_bt); wallet_tools::process_transactions(m_wl_bob.get(), events, m_head, m_bt); + MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get())); + MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get())); } void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events) @@ -804,12 +911,14 @@ void gen_trezor_base::test_setup(std::vector<test_event_entry>& events) add_shared_events(events); setup_trezor(); + update_client_settings(); + m_alice_account.create_from_device(*m_trezor); m_alice_account.set_createtime(m_wallet_ts); - m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true)); - m_wl_eve.reset(new tools::wallet2(MAINNET, 1, true)); + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); @@ -818,13 +927,11 @@ void gen_trezor_base::test_setup(std::vector<test_event_entry>& events) wallet_tools::process_transactions(m_wl_eve.get(), events, m_head, m_bt); } -void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep) +void gen_trezor_base::add_transactions_to_events( + std::vector<test_event_entry>& events, + test_generator &generator, + const std::vector<cryptonote::transaction> &txs) { - // Construct pending transaction for signature in the Trezor. - const uint64_t height_pre = num_blocks(events) - 1; - cryptonote::block head_block = get_head_block(events); - const crypto::hash head_hash = get_block_hash(head_block); - // If current test requires higher hard-fork, move it up const auto current_hf = m_hard_forks.back().first; const uint8_t tx_hf = m_rct_config.bp_version == 2 ? 10 : 9; @@ -832,8 +939,28 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: throw std::runtime_error("Too late for HF change"); } - tools::wallet2::unsigned_tx_set txs; std::list<cryptonote::transaction> tx_list; + for(const auto & tx : txs) + { + events.push_back(tx); + tx_list.push_back(tx); + } + + MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_new, m_head, m_miner_account, tx_list, tx_hf); + MDEBUG("New tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_new)); + + m_head = blk_new; +} + +void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep) +{ + // Construct pending transaction for signature in the Trezor. + const uint64_t height_pre = num_blocks(events) - 1; + cryptonote::block head_block = get_head_block(events); + const crypto::hash head_hash = get_block_hash(head_block); + + tools::wallet2::unsigned_tx_set txs; + std::vector<cryptonote::transaction> tx_list; for(auto &ptx : ptxs) { txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(ptx, *m_trezor)); @@ -848,6 +975,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: hw::wallet_shim wallet_shim; setup_shim(&wallet_shim); aux_data.tx_recipients = dsts_info; + aux_data.bp_version = m_rct_config.bp_version; dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); MDEBUG("Signed tx data from hw: " << exported_txs.ptx.size() << " transactions"); @@ -865,13 +993,11 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: CHECK_AND_ASSERT_THROW_MES(resx, "Trezor tx_1 semantics failed"); CHECK_AND_ASSERT_THROW_MES(resy, "Trezor tx_1 Nonsemantics failed"); - events.push_back(c_ptx.tx); tx_list.push_back(c_ptx.tx); MDEBUG("Transaction: " << dump_data(c_ptx.tx)); } - MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_7, m_head, m_miner_account, tx_list, tx_hf); - MDEBUG("Trezor tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_7)); + add_transactions_to_events(events, generator, tx_list); // TX receive test uint64_t sum_in = 0; @@ -909,7 +1035,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: const bool sender = widx == 0; tools::wallet2 *wl = wallets[widx]; - wallet_tools::process_transactions(wl, events, blk_7, m_bt, boost::make_optional(head_hash)); + wallet_tools::process_transactions(wl, events, m_head, m_bt, boost::make_optional(head_hash)); tools::wallet2::transfer_container m_trans; tools::wallet2::transfer_container m_trans_txid; @@ -972,6 +1098,120 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: } CHECK_AND_ASSERT_THROW_MES(sum_in == sum_out, "Tx amount mismatch"); + + // Test get_tx_key feature for stored private tx keys + test_get_tx(events, wallets, exported_txs.ptx, aux_data.tx_device_aux); +} + +bool gen_trezor_base::verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs) +{ + ::crypto::public_key tx_pub_c; + ::crypto::secret_key_to_public_key(tx_priv, tx_pub_c); + if (tx_pub == tx_pub_c) + return true; + + for(const auto & elem : subs) + { + tx_pub_c = rct::rct2pk(rct::scalarmultKey(rct::pk2rct(elem.first), rct::sk2rct(tx_priv))); + if (tx_pub == tx_pub_c) + return true; + } + return false; +} + +void gen_trezor_base::test_get_tx( + std::vector<test_event_entry>& events, + std::vector<tools::wallet2*> wallets, + const std::vector<tools::wallet2::pending_tx> &ptxs, + const std::vector<std::string> &aux_tx_info) +{ + if (!m_test_get_tx_key) + { + return; + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + if (!dev_cold->is_get_tx_key_supported()) + { + MERROR("Get TX key is not supported by the connected Trezor"); + return; + } + + subaddresses_t all_subs; + for(tools::wallet2 * wlt : wallets) + { + wlt->expand_subaddresses({10, 20}); + + const subaddresses_t & cur_sub = wallet_accessor_test::get_subaddresses(wlt); + all_subs.insert(cur_sub.begin(), cur_sub.end()); + } + + for(size_t txid = 0; txid < ptxs.size(); ++txid) + { + const auto &c_ptx = ptxs[txid]; + const auto &c_tx = c_ptx.tx; + const ::crypto::hash tx_prefix_hash = cryptonote::get_transaction_prefix_hash(c_tx); + + auto tx_pub = cryptonote::get_tx_pub_key_from_extra(c_tx.extra); + auto additional_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(c_tx.extra); + + hw::device_cold:: tx_key_data_t tx_key_data; + std::vector<::crypto::secret_key> tx_keys; + + dev_cold->load_tx_key_data(tx_key_data, aux_tx_info[txid]); + CHECK_AND_ASSERT_THROW_MES(std::string(tx_prefix_hash.data, 32) == tx_key_data.tx_prefix_hash, "TX prefix mismatch"); + + dev_cold->get_tx_key(tx_keys, tx_key_data, m_alice_account.get_keys().m_view_secret_key); + CHECK_AND_ASSERT_THROW_MES(!tx_keys.empty(), "Empty TX keys"); + CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[0], tx_pub, all_subs), "Tx pub mismatch"); + CHECK_AND_ASSERT_THROW_MES(additional_pub_keys.size() == tx_keys.size() - 1, "Invalid additional keys count"); + + for(size_t i = 0; i < additional_pub_keys.size(); ++i) + { + CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[i + 1], additional_pub_keys[i], all_subs), "Tx pub mismatch"); + } + } +} + +void gen_trezor_base::mine_and_test(std::vector<test_event_entry>& events) +{ + cryptonote::core * core = daemon()->core(); + const uint64_t height_before_mining = daemon()->get_height(); + + const auto miner_address = cryptonote::get_account_address_as_str(FAKECHAIN, false, get_address(m_miner_account)); + daemon()->mine_blocks(1, miner_address); + + const uint64_t cur_height = daemon()->get_height(); + CHECK_AND_ASSERT_THROW_MES(height_before_mining < cur_height, "Mining fail"); + + const crypto::hash top_hash = core->get_blockchain_storage().get_block_id_by_height(height_before_mining); + cryptonote::block top_block{}; + CHECK_AND_ASSERT_THROW_MES(core->get_blockchain_storage().get_block_by_hash(top_hash, top_block), "Block fetch fail"); + CHECK_AND_ASSERT_THROW_MES(!top_block.tx_hashes.empty(), "Mined block is empty"); + + std::vector<cryptonote::transaction> txs_found; + std::vector<crypto::hash> txs_missed; + bool r = core->get_blockchain_storage().get_transactions(top_block.tx_hashes, txs_found, txs_missed); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction lookup fail"); + CHECK_AND_ASSERT_THROW_MES(!txs_found.empty(), "Transaction lookup fail"); + + // Transaction is not expanded, but mining verified it. + events.push_back(txs_found[0]); + events.push_back(top_block); +} + +void gen_trezor_base::set_hard_fork(uint8_t hf) +{ + m_top_hard_fork = hf; + if (hf < 9){ + throw std::runtime_error("Minimal supported Hardfork is 9"); + } else if (hf == 9){ + rct_config({rct::RangeProofPaddedBulletproof, 1}); + } else { + rct_config({rct::RangeProofPaddedBulletproof, 2}); + } } #define TREZOR_TEST_PREFIX() \ @@ -1182,6 +1422,48 @@ std::vector<tools::wallet2::pending_tx> tsx_builder::build() return m_ptxs; } +device_trezor_test::device_trezor_test(): m_tx_sign_ctr(0), m_compute_key_image_ctr(0) {} + +void device_trezor_test::clear_test_counters(){ + m_tx_sign_ctr = 0; + m_compute_key_image_ctr = 0; +} + +void device_trezor_test::setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type){ + this->clear_test_counters(); + this->set_callback(nullptr); + this->set_debug(true); // debugging commands on Trezor (auto-confirm transactions) + + CHECK_AND_ASSERT_THROW_MES(this->set_name(trezor_path), "Could not set device name " << trezor_path); + this->set_network_type(network_type); + this->set_derivation_path(""); // empty derivation path + + CHECK_AND_ASSERT_THROW_MES(this->init(), "Could not initialize the device " << trezor_path); + CHECK_AND_ASSERT_THROW_MES(this->connect(), "Could not connect to the device " << trezor_path); + this->wipe_device(); + this->load_device(seed); + this->release(); + this->disconnect(); +} + +bool device_trezor_test::compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key, + const ::crypto::key_derivation &recv_derivation, size_t real_output_index, + const ::cryptonote::subaddress_index &received_index, + ::cryptonote::keypair &in_ephemeral, ::crypto::key_image &ki) { + + bool res = device_trezor::compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, + in_ephemeral, ki); + m_compute_key_image_ctr += res; + return res; +} + +void +device_trezor_test::tx_sign(hw::wallet_shim *wallet, const ::tools::wallet2::unsigned_tx_set &unsigned_tx, size_t idx, + hw::tx_aux_data &aux_data, std::shared_ptr<hw::trezor::protocol::tx::Signer> &signer) { + m_tx_sign_ctr += 1; + device_trezor::tx_sign(wallet, unsigned_tx, idx, aux_data, signer); +} + bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events) { test_generator generator(m_generator); @@ -1207,6 +1489,92 @@ bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events) uint64_t spent = 0, unspent = 0; m_wl_alice->import_key_images(ski, 0, spent, unspent, false); + + auto dev_trezor_test = dynamic_cast<device_trezor_test*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement test interface"); + if (!m_live_refresh_enabled) + CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == 0, "Live refresh should not happen: " << dev_trezor_test->m_compute_key_image_ctr); + else + CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == ski.size(), "Live refresh counts invalid"); + + return true; +} + +bool gen_trezor_ki_sync_with_refresh::generate(std::vector<test_event_entry>& events) +{ + m_live_refresh_enabled = true; + return gen_trezor_ki_sync::generate(events); +} + +bool gen_trezor_ki_sync_without_refresh::generate(std::vector<test_event_entry>& events) +{ + m_live_refresh_enabled = false; + return gen_trezor_ki_sync::generate(events); +} + +bool gen_trezor_live_refresh::generate(std::vector<test_event_entry>& events) +{ + test_generator generator(m_generator); + test_setup(events); + + auto dev_cold = dynamic_cast<::hw::device_cold*>(m_trezor); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + if (!dev_cold->is_live_refresh_supported()){ + MDEBUG("Trezor does not support live refresh"); + return true; + } + + hw::device & sw_device = hw::get_device("default"); + + dev_cold->live_refresh_start(); + for(unsigned i=0; i<50; ++i) + { + cryptonote::subaddress_index subaddr = {0, i}; + + ::crypto::secret_key r; + ::crypto::public_key R; + ::crypto::key_derivation D; + ::crypto::public_key pub_ver; + ::crypto::key_image ki; + + ::crypto::random32_unbiased((unsigned char*)r.data); + ::crypto::secret_key_to_public_key(r, R); + memcpy(D.data, rct::scalarmultKey(rct::pk2rct(R), rct::sk2rct(m_alice_account.get_keys().m_view_secret_key)).bytes, 32); + + ::crypto::secret_key scalar_step1; + ::crypto::secret_key scalar_step2; + ::crypto::derive_secret_key(D, i, m_alice_account.get_keys().m_spend_secret_key, scalar_step1); + if (i == 0) + { + scalar_step2 = scalar_step1; + } + else + { + ::crypto::secret_key subaddr_sk = sw_device.get_subaddress_secret_key(m_alice_account.get_keys().m_view_secret_key, subaddr); + sw_device.sc_secret_add(scalar_step2, scalar_step1, subaddr_sk); + } + + ::crypto::secret_key_to_public_key(scalar_step2, pub_ver); + ::crypto::generate_key_image(pub_ver, scalar_step2, ki); + + cryptonote::keypair in_ephemeral; + ::crypto::key_image ki2; + + dev_cold->live_refresh( + m_alice_account.get_keys().m_view_secret_key, + pub_ver, + D, + i, + subaddr, + in_ephemeral, + ki2 + ); + + CHECK_AND_ASSERT_THROW_MES(ki == ki2, "Key image inconsistent"); + } + + dev_cold->live_refresh_finish(); return true; } @@ -1407,3 +1775,62 @@ bool gen_trezor_many_utxo::generate(std::vector<test_event_entry>& events) TREZOR_TEST_SUFFIX(); } +void wallet_api_tests::init() +{ + m_wallet_dir = boost::filesystem::unique_path(); + boost::filesystem::create_directories(m_wallet_dir); +} + +wallet_api_tests::~wallet_api_tests() +{ + try + { + if (!m_wallet_dir.empty() && boost::filesystem::exists(m_wallet_dir)) + { + boost::filesystem::remove_all(m_wallet_dir); + } + } + catch(...) + { + MERROR("Could not remove wallet directory"); + } +} + +bool wallet_api_tests::generate(std::vector<test_event_entry>& events) +{ + init(); + test_setup(events); + const std::string wallet_path = (m_wallet_dir / "wallet").string(); + const auto api_net_type = m_network_type == TESTNET ? Monero::TESTNET : Monero::MAINNET; + + Monero::WalletManager *wmgr = Monero::WalletManagerFactory::getWalletManager(); + std::unique_ptr<Monero::Wallet> w{wmgr->createWalletFromDevice(wallet_path, "", api_net_type, m_trezor_path, 1)}; + CHECK_AND_ASSERT_THROW_MES(w->init(daemon()->rpc_addr(), 0), "Wallet init fail"); + CHECK_AND_ASSERT_THROW_MES(w->refresh(), "Refresh fail"); + uint64_t balance = w->balance(0); + MINFO("Balance: " << balance); + CHECK_AND_ASSERT_THROW_MES(w->status() == Monero::PendingTransaction::Status_Ok, "Status nok"); + + auto addr = get_address(m_eve_account); + auto recepient_address = cryptonote::get_account_address_as_str(m_network_type, false, addr); + Monero::PendingTransaction * transaction = w->createTransaction(recepient_address, + "", + MK_COINS(10), + TREZOR_TEST_MIXIN, + Monero::PendingTransaction::Priority_Medium, + 0, + std::set<uint32_t>{}); + CHECK_AND_ASSERT_THROW_MES(transaction->status() == Monero::PendingTransaction::Status_Ok, "Status nok"); + w->refresh(); + + CHECK_AND_ASSERT_THROW_MES(w->balance(0) == balance, "Err"); + CHECK_AND_ASSERT_THROW_MES(transaction->amount() == MK_COINS(10), "Err"); + CHECK_AND_ASSERT_THROW_MES(transaction->commit(), "Err"); + CHECK_AND_ASSERT_THROW_MES(w->balance(0) != balance, "Err"); + CHECK_AND_ASSERT_THROW_MES(wmgr->closeWallet(w.get()), "Err"); + (void)w.release(); + + mine_and_test(events); + return true; +} + diff --git a/tests/trezor/trezor_tests.h b/tests/trezor/trezor_tests.h index 41db1cce5..bed49fec4 100644 --- a/tests/trezor/trezor_tests.h +++ b/tests/trezor/trezor_tests.h @@ -31,6 +31,8 @@ #pragma once #include <device_trezor/device_trezor.hpp> +#include <wallet/api/wallet2_api.h> +#include "daemon.h" #include "../core_tests/chaingen.h" #include "../core_tests/wallet_tools.h" @@ -50,29 +52,48 @@ public: gen_trezor_base(const gen_trezor_base &other); virtual ~gen_trezor_base() {}; - void setup_args(const std::string & trezor_path, bool heavy_tests=false); + virtual void setup_args(const std::string & trezor_path, bool heavy_tests=false); virtual bool generate(std::vector<test_event_entry>& events); virtual void load(std::vector<test_event_entry>& events); // load events, init test obj - void fix_hf(std::vector<test_event_entry>& events); - void update_trackers(std::vector<test_event_entry>& events); + virtual void fix_hf(std::vector<test_event_entry>& events); + virtual void update_trackers(std::vector<test_event_entry>& events); - void fork(gen_trezor_base & other); // fork generated chain to another test - void clear(); // clears m_events, bt, generator, hforks - void add_shared_events(std::vector<test_event_entry>& events); // m_events -> events - void test_setup(std::vector<test_event_entry>& events); // init setup env, wallets + virtual void fork(gen_trezor_base & other); // fork generated chain to another test + virtual void clear(); // clears m_events, bt, generator, hforks + virtual void add_shared_events(std::vector<test_event_entry>& events); // m_events -> events + virtual void test_setup(std::vector<test_event_entry>& events); // init setup env, wallets - void test_trezor_tx(std::vector<test_event_entry>& events, + virtual void add_transactions_to_events( + std::vector<test_event_entry>& events, + test_generator &generator, + const std::vector<cryptonote::transaction> &txs); + + virtual void test_trezor_tx( + std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep=false); - crypto::hash head_hash() { return get_block_hash(m_head); } - cryptonote::block head_block() { return m_head; } - bool heavy_tests() { return m_heavy_tests; } + virtual void test_get_tx( + std::vector<test_event_entry>& events, + std::vector<tools::wallet2*> wallets, + const std::vector<tools::wallet2::pending_tx> &ptxs, + const std::vector<std::string> &aux_tx_info); + + virtual void mine_and_test(std::vector<test_event_entry>& events); + + virtual void set_hard_fork(uint8_t hf); + + crypto::hash head_hash() const { return get_block_hash(m_head); } + cryptonote::block head_block() const { return m_head; } + bool heavy_tests() const { return m_heavy_tests; } void rct_config(rct::RCTConfig rct_config) { m_rct_config = rct_config; } - uint8_t cur_hf(){ return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; } + uint8_t cur_hf() const { return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; } + cryptonote::network_type nettype() const { return m_network_type; } + std::shared_ptr<mock_daemon> daemon() const { return m_daemon; } + void daemon(std::shared_ptr<mock_daemon> daemon){ m_daemon = std::move(daemon); } // Static configuration static const uint64_t m_ts_start; @@ -84,19 +105,26 @@ public: static const std::string m_alice_view_private; protected: - void setup_trezor(); - void init_fields(); + virtual void setup_trezor(); + virtual void init_fields(); + virtual void update_client_settings(); + virtual bool verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs); test_generator m_generator; block_tracker m_bt; + cryptonote::network_type m_network_type; + std::shared_ptr<mock_daemon> m_daemon; + uint8_t m_top_hard_fork; v_hardforks_t m_hard_forks; cryptonote::block m_head; std::vector<test_event_entry> m_events; std::string m_trezor_path; bool m_heavy_tests; + bool m_test_get_tx_key; rct::RCTConfig m_rct_config; + bool m_live_refresh_enabled; cryptonote::account_base m_miner_account; cryptonote::account_base m_bob_account; @@ -113,6 +141,7 @@ protected: void serialize(Archive & ar, const unsigned int /*version*/) { ar & m_generator; + ar & m_network_type; } }; @@ -169,12 +198,52 @@ protected: rct::RCTConfig m_rct_config; }; +// Trezor device ship to track actual method calls. +class device_trezor_test : public hw::trezor::device_trezor { +public: + size_t m_tx_sign_ctr; + size_t m_compute_key_image_ctr; + + device_trezor_test(); + + void clear_test_counters(); + void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type); + + bool compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key, + const ::crypto::key_derivation &recv_derivation, size_t real_output_index, + const ::cryptonote::subaddress_index &received_index, ::cryptonote::keypair &in_ephemeral, + ::crypto::key_image &ki) override; + +protected: + void tx_sign(hw::wallet_shim *wallet, const ::tools::wallet2::unsigned_tx_set &unsigned_tx, size_t idx, + hw::tx_aux_data &aux_data, std::shared_ptr<hw::trezor::protocol::tx::Signer> &signer) override; +}; + +// Tests class gen_trezor_ki_sync : public gen_trezor_base { public: bool generate(std::vector<test_event_entry>& events) override; }; +class gen_trezor_ki_sync_with_refresh : public gen_trezor_ki_sync +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + +class gen_trezor_ki_sync_without_refresh : public gen_trezor_ki_sync +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + +class gen_trezor_live_refresh : public gen_trezor_base +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + class gen_trezor_1utxo : public gen_trezor_base { public: @@ -246,3 +315,15 @@ class gen_trezor_many_utxo : public gen_trezor_base public: bool generate(std::vector<test_event_entry>& events) override; }; + +// Wallet::API tests +class wallet_api_tests : public gen_trezor_base +{ +public: + virtual ~wallet_api_tests(); + void init(); + bool generate(std::vector<test_event_entry>& events) override; + +protected: + boost::filesystem::path m_wallet_dir; +};
\ No newline at end of file diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index b355d566d..bfaea6add 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -52,10 +52,12 @@ set(unit_tests_sources json_serialization.cpp get_xtype_from_string.cpp hashchain.cpp + hmac_keccak.cpp http.cpp keccak.cpp logging.cpp long_term_block_weight.cpp + lmdb.cpp main.cpp memwipe.cpp mlocker.cpp @@ -101,6 +103,7 @@ target_link_libraries(unit_tests cryptonote_protocol cryptonote_core blockchain_db + lmdb_lib rpc net serialization diff --git a/tests/unit_tests/hmac_keccak.cpp b/tests/unit_tests/hmac_keccak.cpp new file mode 100644 index 000000000..cb35d272a --- /dev/null +++ b/tests/unit_tests/hmac_keccak.cpp @@ -0,0 +1,152 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "gtest/gtest.h" +#include "../io.h" + +extern "C" { +#include "crypto/hmac-keccak.h" +} + +#define KECCAK_BLOCKLEN 136 + +static const struct { + const char *key; + const char *inp; + const char *res; +} keccak_hmac_vectors[] = { + {"", + "", + "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b", + }, + {"00", + "", + "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b", + }, + {"00000000000000000000000000000000", + "", + "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b", + }, + {"", + "00", + "402541d5d678ad30217023a12efbd2f367388dc0253ccddb8f915d187e03d414", + }, + {"0000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "d164c38ca454176281b3c09dc83cf70bd00290835ff5490356993d368cde0577", + }, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0000000000000000000000000000000000000000000000000000000000000000", + "d52c9100301a9546d8c97d7f32e8178fd5f4b0a7646ab761db9e0f503f8b0542", + }, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "d52c9100301a9546d8c97d7f32e8178fd5f4b0a7646ab761db9e0f503f8b0542", + "5b9d98c5d3249176230d70fdb215ee2f93e4783064c9564384ddc396e63c18cb", + }, + {"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "98d033ba0c95bb3aaf5709733443cbedba20f6451b710ccba4118fc5523013c8", + }, + {"65768798a9bacbdcedfe0f2031425364758697a8b9cadbecfd0e1f30415263748596a7b8c9daebfc0d1e2f405162738495a6b7c8d9eafb0c1d2e3f5061728394a5b6c7d8e9fa0b1c2d3e4f60718293a4b5c6d7e8f90a1b2c3d4e5f708192a3b4c5d6e7f8091a2b3c4d5e6f8091a2b3c4d5e6f708192a3b4c5d6e7f90a1b2c3d4e5f60718293a4b5c6d7e8fa0b1c2d3e4f5061728394a5b6c7d8e9fb0c1d2e3f405162738495a6b7c8d9eafc0d1e2f30415263748596a7b8c9daebfd0e1f2031425364758697a8b9cadbecfe0f102132435465768798a9bacbdcedff00112233445566778899aabbccddeef00112233445566778899aabbccddeeff102132435465768798a9bacbdcedfe0f2031425364758697a8b9cadbecfd0e1f30415263748596a7b8c9daebfc0d1e2f405162738495a6b7c8d9eafb0c1d2e3f5061728394", + "7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b2835424f5c697683909daab7c4d1deebf805121f2c394653606d7a8794a1aebbc8d5e2effc091623303d4a5764717e8b98a5b2bfccd9e6f3000d1a2734414e5b6875828f9ca9b6c3d0ddeaf704111e2b3845525f6c798693a0adbac7d4e1eefb0815222f3c495663707d8a97a4b1becbd8e5f2ff0c192633404d5a6774818e9ba8b5c2cfdce9f603101d2a3744515e6b7885929facb9c6d3e0edfa0714212e3b4855626f7c8996a3b0bdcad7e4f1fe0b1825323f4c596673808d9aa7b4c1cedbe8f5020f1c293643505d6a7784919eabb8c5d2dfecf90613202d3a4754616e7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b2835424f5c697683909daab7c4d1deebf805121f2c394653606d7a8794a1ae", + "117b60a7f474fd2245adff82a69429367de8f5263fa96c411986009a743b0e70", + }, + {"00112233445566778899aabbccddeeff102132435465768798a9bacbdcedfe0f", + "000d1a2734414e5b6875828f9ca9b6c3d0ddeaf704111e2b3845525f6c798693a0adbac7d4e1eefb0815222f3c495663707d8a97a4b1becbd8e5f2ff0c192633404d5a6774818e9ba8b5c2cfdce9f603101d2a3744515e6b7885929facb9c6d3e0edfa0714212e3b4855626f7c8996a3b0bdcad7e4f1fe0b1825323f4c596673808d9aa7b4c1cedbe8f5020f1c293643505d6a7784919eabb8c5d2dfecf90613202d3a4754616e7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b", + "cbe3ca627c6432c9a66f07c6411fccca894c9b083b55d0773152460142a676ed", + }, +}; + +static void test_keccak_hmac(const size_t * chunks) +{ + uint8_t inp_buff[1024]; + uint8_t key_buff[1024]; + uint8_t res_exp[32]; + uint8_t res_comp[32]; + const size_t len_chunks = chunks ? sizeof(chunks) / sizeof(*chunks) : 0; + + for (size_t i = 0; i < (sizeof(keccak_hmac_vectors) / sizeof(*keccak_hmac_vectors)); i++) + { + const size_t inp_len = strlen(keccak_hmac_vectors[i].inp)/2; + const size_t key_len = strlen(keccak_hmac_vectors[i].key)/2; + ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].inp, inp_len, inp_buff)); + ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].key, key_len, key_buff)); + ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].res, 32, res_exp)); + if (len_chunks == 0) + { + hmac_keccak_hash(res_comp, key_buff, key_len, inp_buff, inp_len); + } + else + { + hmac_keccak_state S; + hmac_keccak_init(&S, key_buff, key_len); + size_t inp_passed = 0; + size_t chunk_size = 0; + while(inp_passed < inp_len){ + const size_t to_pass = std::min(inp_len - inp_passed, chunks[chunk_size]); + hmac_keccak_update(&S, inp_buff + inp_passed, to_pass); + + inp_passed += to_pass; + chunk_size = (chunk_size + 1) % len_chunks; + } + + hmac_keccak_finish(&S, res_comp); + } + ASSERT_EQ(memcmp(res_exp, res_comp, 32), 0); + } +} + + +TEST(keccak_hmac, ) +{ + test_keccak_hmac({}); +} + +TEST(keccak_hmac, 1) +{ + static const size_t chunks[] = {1}; + test_keccak_hmac(chunks); +} + +TEST(keccak_hmac, 1_20) +{ + static const size_t chunks[] = {1, 20}; + test_keccak_hmac(chunks); +} + +TEST(keccak_hmac, 136_1) +{ + static const size_t chunks[] = {136, 1}; + test_keccak_hmac(chunks); +} + +TEST(keccak_hmac, 137_1) +{ + static const size_t chunks[] = {137, 1}; + test_keccak_hmac(chunks); +} diff --git a/tests/unit_tests/lmdb.cpp b/tests/unit_tests/lmdb.cpp new file mode 100644 index 000000000..c37c83a32 --- /dev/null +++ b/tests/unit_tests/lmdb.cpp @@ -0,0 +1,404 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/algorithm_ext/iota.hpp> +#include <boost/range/algorithm/equal.hpp> +#include <gtest/gtest.h> + +#include "lmdb/database.h" +#include "lmdb/table.h" +#include "lmdb/transaction.h" +#include "lmdb/util.h" + +namespace +{ + enum class choice : unsigned {}; + enum class big_choice : unsigned long {}; + + struct bytes { + char data[16]; + }; + + MONERO_CURSOR(test_cursor); + + template<typename T> + int run_compare(T left, T right, MDB_cmp_func* cmp) + { + MDB_val left_val = lmdb::to_val(left); + MDB_val right_val = lmdb::to_val(right); + return (*cmp)(&left_val, &right_val); + } +} + +TEST(LMDB, Traits) +{ + EXPECT_TRUE((std::is_same<void, lmdb::identity<void>::type>())); + EXPECT_TRUE((std::is_same<unsigned, lmdb::identity<unsigned>::type>())); + + EXPECT_TRUE((std::is_same<void, lmdb::native_type<void>>())); + EXPECT_TRUE((std::is_same<unsigned, lmdb::native_type<unsigned>>())); + EXPECT_TRUE((std::is_same<unsigned, lmdb::native_type<choice>>())); + EXPECT_TRUE((std::is_same<unsigned long, lmdb::native_type<big_choice>>())); +} + +TEST(LMDB, ToNative) +{ + enum class negative_choice : int {}; + + EXPECT_TRUE((std::is_same<unsigned, decltype(lmdb::to_native(choice(0)))>())); + EXPECT_TRUE( + (std::is_same<unsigned long, decltype(lmdb::to_native(big_choice(0)))>()) + ); + EXPECT_TRUE( + (std::is_same<int, decltype(lmdb::to_native(negative_choice(0)))>()) + ); + + EXPECT_EQ(unsigned(0), lmdb::to_native(choice(0))); + EXPECT_EQ(unsigned(0xffffffff), lmdb::to_native(choice(0xffffffff))); + EXPECT_EQ(-1, lmdb::to_native(negative_choice(-1))); + + // test constexpr + static_assert(100 == lmdb::to_native(choice(100)), "to_native failed"); + static_assert(-100 == lmdb::to_native(negative_choice(-100)), "to_native failed"); +} + +TEST(LMDB, Conversions) +{ + struct one + { + big_choice i; + choice j; + }; + + const one test{big_choice(100), choice(95)}; + one test2{big_choice(1000), choice(950)}; + + EXPECT_EQ(&test, lmdb::to_val(test).mv_data); + EXPECT_NE(&test2, lmdb::to_val(test).mv_data); + EXPECT_EQ( + &test, + static_cast<const void*>(lmdb::to_byte_span(lmdb::to_val(test)).begin()) + ); + EXPECT_EQ(sizeof(test), lmdb::to_val(test).mv_size); + EXPECT_EQ(sizeof(test), lmdb::to_byte_span(lmdb::to_val(test)).size()); + + EXPECT_EQ(&test2, lmdb::to_val(test2).mv_data); + EXPECT_NE(&test, lmdb::to_val(test2).mv_data); + EXPECT_EQ( + &test2, + static_cast<const void*>(lmdb::to_byte_span(lmdb::to_val(test2)).begin()) + ); + EXPECT_EQ(sizeof(test2), lmdb::to_val(test2).mv_size); + EXPECT_EQ(sizeof(test2), lmdb::to_byte_span(lmdb::to_val(test2)).size()); +} + +TEST(LMDB, LessSort) +{ + struct one + { + unsigned i; + unsigned j; + }; + + struct two + { + unsigned i; + choice j; + }; + + EXPECT_EQ(0, run_compare(0u, 0u, &lmdb::less<unsigned>)); + EXPECT_EQ(-1, run_compare(0u, 1u, &lmdb::less<unsigned>)); + EXPECT_EQ(1, run_compare(1u, 0u, &lmdb::less<unsigned>)); + + EXPECT_EQ(0, run_compare<one>({0, 1}, {0, 1}, &lmdb::less<unsigned, sizeof(unsigned)>)); + EXPECT_EQ(-1, run_compare<one>({0, 0}, {0, 1}, &lmdb::less<unsigned, sizeof(unsigned)>)); + EXPECT_EQ(1, run_compare<one>({0, 1}, {0, 0}, &lmdb::less<unsigned, sizeof(unsigned)>)); + + EXPECT_EQ(0, run_compare<one>({0, 1}, {0, 1}, MONERO_SORT_BY(one, j))); + EXPECT_EQ(-1, run_compare<one>({0, 0}, {0, 1}, MONERO_SORT_BY(one, j))); + EXPECT_EQ(1, run_compare<one>({0, 1}, {0, 0}, MONERO_SORT_BY(one, j))); + + EXPECT_EQ(0, run_compare<two>({0, choice(1)}, {0, choice(1)}, MONERO_SORT_BY(two, j))); + EXPECT_EQ(-1, run_compare<two>({0, choice(0)}, {0, choice(1)}, MONERO_SORT_BY(two, j))); + EXPECT_EQ(1, run_compare<two>({0, choice(1)}, {0, choice(0)}, MONERO_SORT_BY(two, j))); + + // compare function addresses + EXPECT_EQ((MONERO_SORT_BY(one, i)), (MONERO_SORT_BY(two, i))); + EXPECT_EQ((MONERO_SORT_BY(one, j)), (MONERO_SORT_BY(two, j))); + EXPECT_NE((MONERO_SORT_BY(one, i)), (MONERO_SORT_BY(two, j))); + EXPECT_NE((MONERO_SORT_BY(one, j)), (MONERO_SORT_BY(two, i))); +} + +TEST(LMDB, SortCompare) +{ + struct one + { + unsigned i; + bytes j; + }; + + one test{55}; + boost::iota(test.j.data, 10); + + const one test2 = test; + + EXPECT_EQ(0, run_compare(test, test2, MONERO_COMPARE(one, j))); + + test.j.data[15] = 1; + EXPECT_GT(0, run_compare(test, test2, MONERO_COMPARE(one, j))); + + test.j.data[15] = 100; + EXPECT_LT(0, run_compare(test, test2, MONERO_COMPARE(one, j))); +} + +TEST(LMDB, Table) +{ + struct one + { + bytes i; + bytes j; + }; + + constexpr lmdb::basic_table<choice, bytes> test{"foo"}; + + EXPECT_STREQ("foo", test.name); + static_assert(test.flags == 0, "bad flags"); + static_assert(&lmdb::less<unsigned> == test.key_cmp, "bad key_cmp"); + static_assert(test.value_cmp == nullptr, "bad value_cmp"); + EXPECT_TRUE(test.get_value<bytes>(MDB_val{}).matches(std::errc::invalid_argument)); + + lmdb::basic_table<big_choice, one> test2{ + "foo2", MDB_DUPSORT, &lmdb::compare<one> + }; + + EXPECT_STREQ("foo2", test2.name); + EXPECT_EQ((MDB_DUPSORT | MDB_DUPFIXED), test2.flags); + EXPECT_EQ(&lmdb::less<unsigned long>, test2.key_cmp); + EXPECT_EQ(&lmdb::compare<one>, test2.value_cmp); + EXPECT_TRUE(test2.get_value<one>(MDB_val{}).matches(std::errc::invalid_argument)); + + one record{}; + boost::iota(record.i.data, 0); + boost::iota(record.i.data, 20); + + const one record_copy = MONERO_UNWRAP(test2.get_value<one>(lmdb::to_val(record))); + EXPECT_TRUE(boost::equal(record.i.data, record_copy.i.data)); + EXPECT_TRUE(boost::equal(record.j.data, record_copy.j.data)); + + const bytes j_copy = MONERO_UNWRAP( + test2.get_value<MONERO_FIELD(one, j)>(lmdb::to_val(record)) + ); + EXPECT_TRUE(boost::equal(record.j.data, j_copy.data)); + + EXPECT_TRUE( + test.get_key_stream(test_cursor{}).matches(std::errc::invalid_argument) + ); + EXPECT_TRUE( + test2.get_key_stream(test_cursor{}).matches(std::errc::invalid_argument) + ); + + + EXPECT_TRUE( + test.get_value_stream(choice(0), test_cursor{}).matches(std::errc::invalid_argument) + ); + EXPECT_TRUE( + test2.get_value_stream(big_choice(0), test_cursor{}).matches(std::errc::invalid_argument) + ); +} + +TEST(LMDB, InvalidDatabase) +{ + lmdb::database test{lmdb::environment{}}; + + EXPECT_TRUE(test.resize().matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.create_read_txn().matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.reset_txn(lmdb::read_txn{}).matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.create_write_txn().matches(std::errc::invalid_argument)); + EXPECT_TRUE(test.commit(lmdb::write_txn{}).matches(std::errc::invalid_argument)); + + EXPECT_TRUE( + test.try_write( [](MDB_txn&) { return success(); } ).matches(std::errc::invalid_argument) + ); +} + +TEST(LMDB, InvalidValueStream) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + lmdb::value_stream<one, close_test_cursor> test{test_cursor{}}; + + EXPECT_TRUE((std::is_same<one, decltype(*(test.make_iterator()))>())); + EXPECT_TRUE((std::is_same<one, decltype(*(test.make_range().begin()))>())); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test.make_iterator<MONERO_FIELD(one, k)>()))>()) + ); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test.make_range<MONERO_FIELD(one, k)>().begin()))>()) + ); + + EXPECT_NO_THROW(test.reset()); + EXPECT_EQ(0u, test.count()); + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); + + EXPECT_EQ(0u, test.count()); + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); +} + +TEST(LMDB, InvalidValueIterator) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + lmdb::value_iterator<one> test1{}; + + EXPECT_TRUE((std::is_same<one, decltype(*test1)>())); + EXPECT_TRUE( + (std::is_same<bytes, decltype(test1.get_value<MONERO_FIELD(one, k)>())>()) + ); + + EXPECT_TRUE(test1.is_end()); + EXPECT_NO_THROW(++test1); + EXPECT_NO_THROW(test1++); + EXPECT_TRUE(test1.is_end()); + + lmdb::value_iterator<one> test2{nullptr}; + + EXPECT_TRUE(test2.is_end()); + EXPECT_NO_THROW(++test2); + EXPECT_NO_THROW(test2++); + EXPECT_TRUE(test2.is_end()); + + EXPECT_TRUE(test1.equal(test2)); + EXPECT_TRUE(test2.equal(test1)); + EXPECT_TRUE(test1 == test2); + EXPECT_TRUE(test2 == test1); + EXPECT_FALSE(test1 != test2); + EXPECT_FALSE(test2 != test1); + + lmdb::value_iterator<MONERO_FIELD(one, k)> test3{}; + + EXPECT_TRUE((std::is_same<bytes, decltype(*test3)>())); + EXPECT_TRUE((std::is_same<one, decltype(test3.get_value<one>())>())); + EXPECT_TRUE( + (std::is_same<choice, decltype(test1.get_value<MONERO_FIELD(one, j)>())>()) + ); + + EXPECT_TRUE(test3.is_end()); + EXPECT_NO_THROW(++test3); + EXPECT_NO_THROW(test3++); + EXPECT_TRUE(test3.is_end()); +} + +TEST(LMDB, InvalidKeyStream) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + using record = std::pair<choice, boost::iterator_range<lmdb::value_iterator<one>>>; + + lmdb::key_stream<choice, one, close_test_cursor> test{test_cursor{}}; + + EXPECT_TRUE((std::is_same<record, decltype(*(test.make_iterator()))>())); + EXPECT_TRUE((std::is_same<record, decltype(*(test.make_range().begin()))>())); + + EXPECT_NO_THROW(test.reset()); + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); + + EXPECT_TRUE(test.make_iterator().is_end()); + EXPECT_TRUE(test.make_range().empty()); + EXPECT_EQ(nullptr, test.give_cursor()); +} + +TEST(LMDB, InvalidKeyIterator) +{ + struct one + { + choice i; + choice j; + bytes k; + }; + + using record = std::pair<choice, boost::iterator_range<lmdb::value_iterator<one>>>; + + lmdb::key_iterator<choice, one> test1{}; + + EXPECT_TRUE((std::is_same<record, decltype(*test1)>())); + EXPECT_TRUE((std::is_same<choice, decltype(test1.get_key())>())); + EXPECT_TRUE((std::is_same<one, decltype(*(test1.make_value_iterator()))>())); + EXPECT_TRUE((std::is_same<one, decltype(*(test1.make_value_range().begin()))>())); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test1.make_value_iterator<MONERO_FIELD(one, k)>()))>()) + ); + EXPECT_TRUE( + (std::is_same<bytes, decltype(*(test1.make_value_range<MONERO_FIELD(one, k)>().begin()))>()) + ); + + EXPECT_TRUE(test1.is_end()); + EXPECT_NO_THROW(++test1); + EXPECT_NO_THROW(test1++); + EXPECT_TRUE(test1.is_end()); + EXPECT_TRUE(test1.make_value_iterator().is_end()); + EXPECT_TRUE(test1.make_value_range().empty()); + + lmdb::key_iterator<choice, one> test2{nullptr}; + + EXPECT_TRUE(test2.is_end()); + EXPECT_NO_THROW(++test2); + EXPECT_NO_THROW(test2++); + EXPECT_TRUE(test2.is_end()); + EXPECT_TRUE(test2.make_value_iterator().is_end()); + EXPECT_TRUE(test2.make_value_range().empty()); + + EXPECT_TRUE(test1.equal(test2)); + EXPECT_TRUE(test2.equal(test1)); + EXPECT_TRUE(test1 == test2); + EXPECT_TRUE(test2 == test1); + EXPECT_FALSE(test1 != test2); + EXPECT_FALSE(test2 != test1); +} + + diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp index f37b608b6..ce6ff3551 100644 --- a/tests/unit_tests/long_term_block_weight.cpp +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -64,6 +64,18 @@ public: virtual uint64_t height() const override { return blocks.size(); } virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight); + return ret; + } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override { + std::vector<uint64_t> ret; + ret.reserve(count); + while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight); + return ret; + } virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override { uint64_t h = height(); crypto::hash top = crypto::null_hash; |