diff options
Diffstat (limited to 'src')
71 files changed, 1190 insertions, 662 deletions
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 7118b0881..7870a2efd 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -30,7 +30,6 @@ #pragma once -#include <list> #include <string> #include <exception> #include <boost/program_options.hpp> @@ -1307,11 +1306,11 @@ public: * get_output_data(const uint64_t& amount, const uint64_t& index) * but for a list of outputs rather than just one. * - * @param amount an output amount + * @param amounts an output amount, or as many as offsets * @param offsets a list of amount-specific output indices * @param outputs return-by-reference a list of outputs' metadata */ - virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) = 0; + virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) = 0; /* * FIXME: Need to check with git blame and ask what this does to diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ea3638a85..5af5d1903 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -29,10 +29,8 @@ #include <boost/filesystem.hpp> #include <boost/format.hpp> -#include <boost/current_function.hpp> #include <memory> // std::unique_ptr #include <cstring> // memcpy -#include <random> #include "string_tools.h" #include "file_io_utils.h" @@ -1350,6 +1348,15 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) #if VERSION > 0 else if (db_version < VERSION) { + if (mdb_flags & MDB_RDONLY) + { + txn.abort(); + mdb_env_close(m_env); + m_open = false; + MFATAL("Existing lmdb database needs to be converted, which cannot be done on a read-only database."); + MFATAL("Please run monerod once to convert the database."); + return; + } // Note that there was a schema change within version 0 as well. // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. @@ -3197,8 +3204,11 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 TXN_POSTFIX_RDONLY(); } -void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial) +void BlockchainLMDB::get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial) { + if (amounts.size() != 1 && amounts.size() != offsets.size()) + throw0(DB_ERROR("Invalid sizes of amounts and offets")); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); TIME_MEASURE_START(db3); check_open(); @@ -3209,10 +3219,11 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui RCURSOR(output_amounts); - MDB_val_set(k, amount); - for (const uint64_t &index : offsets) + for (size_t i = 0; i < offsets.size(); ++i) { - MDB_val_set(v, index); + const uint64_t amount = amounts.size() == 1 ? amounts[0] : amounts[i]; + MDB_val_set(k, amount); + MDB_val_set(v, offsets[i]); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) @@ -3222,7 +3233,7 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui MDEBUG("Partial result: " << outputs.size() << "/" << offsets.size()); break; } - throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(index) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str())); + throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(offsets[i]) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str())); } else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); @@ -4358,16 +4369,12 @@ void BlockchainLMDB::migrate_2_3() void BlockchainLMDB::migrate(const uint32_t oldversion) { - switch(oldversion) { - case 0: - migrate_0_1(); /* FALLTHRU */ - case 1: - migrate_1_2(); /* FALLTHRU */ - case 2: - migrate_2_3(); /* FALLTHRU */ - default: - ; - } + if (oldversion < 1) + migrate_0_1(); + if (oldversion < 2) + migrate_1_2(); + if (oldversion < 3) + migrate_2_3(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 7e76236a5..26159ab4d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -243,7 +243,7 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); + virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 716b33cae..aae8f333b 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -234,7 +234,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' } time_t tt = blk.timestamp; char timebuf[64]; - gmtime_r(&tt, &currtm); + epee::misc_utils::get_gmt_time(tt, currtm); if (!prevtm.tm_year) prevtm = currtm; // catch change of day diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index 6251fcc91..1807d44d9 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -28,17 +28,15 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#include "include_base_utils.h" - -using namespace epee; - #include "checkpoints.h" #include "common/dns_utils.h" -#include "include_base_utils.h" #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include <vector> + +using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "checkpoints" diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h index 61be2c27a..ad2b44d1a 100644 --- a/src/checkpoints/checkpoints.h +++ b/src/checkpoints/checkpoints.h @@ -30,7 +30,6 @@ #pragma once #include <map> -#include <vector> #include "misc_log_ex.h" #include "crypto/hash.h" #include "cryptonote_config.h" diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index aed9bfee7..5da23c944 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -43,7 +43,8 @@ set(common_sources spawn.cpp threadpool.cpp updates.cpp - aligned.c) + aligned.c + combinator.cpp) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) @@ -62,7 +63,6 @@ set(common_private_headers error.h expect.h http_connection.h - int-util.h notify.h pod-class.h rpc_client.h @@ -77,7 +77,8 @@ set(common_private_headers stack_trace.h threadpool.h updates.h - aligned.h) + aligned.h + combinator.h) monero_private_headers(common ${common_private_headers}) diff --git a/src/common/base58.cpp b/src/common/base58.cpp index b28a04f20..cd2272356 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -36,7 +36,6 @@ #include "crypto/hash.h" #include "int-util.h" -#include "util.h" #include "varint.h" namespace tools diff --git a/src/common/combinator.cpp b/src/common/combinator.cpp new file mode 100644 index 000000000..cb4fbc908 --- /dev/null +++ b/src/common/combinator.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "combinator.h" + +namespace tools { + +uint64_t combinations_count(uint32_t k, uint32_t n) +{ + if (k > n) { + throw std::runtime_error("k must not be greater than n"); + } + + uint64_t c = 1; + for (uint64_t i = 1; i <= k; ++i) { + c *= n--; + c /= i; + } + + return c; +} + +} diff --git a/src/common/combinator.h b/src/common/combinator.h new file mode 100644 index 000000000..72c6800d5 --- /dev/null +++ b/src/common/combinator.h @@ -0,0 +1,96 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include <iostream> +#include <vector> + +namespace tools { + +uint64_t combinations_count(uint32_t k, uint32_t n); + +template<typename T> +class Combinator { +public: + Combinator(const std::vector<T>& v) : origin(v) { } + + std::vector<std::vector<T>> combine(size_t k); + +private: + void doCombine(size_t from, size_t k); + + std::vector<T> origin; + std::vector<std::vector<T>> combinations; + std::vector<size_t> current; +}; + +template<typename T> +std::vector<std::vector<T>> Combinator<T>::combine(size_t k) +{ + if (k > origin.size()) + { + throw std::runtime_error("k must be smaller than elements number"); + } + + if (k == 0) + { + throw std::runtime_error("k must be greater than zero"); + } + + combinations.clear(); + doCombine(0, k); + return combinations; +} + +template<typename T> +void Combinator<T>::doCombine(size_t from, size_t k) +{ + current.push_back(0); + + for (size_t i = from; i <= origin.size() - k; ++i) + { + current.back() = i; + + if (k > 1) { + doCombine(i + 1, k - 1); + } else { + std::vector<T> comb; + for (auto ind: current) { + comb.push_back(origin[ind]); + } + combinations.push_back(comb); + } + } + + current.pop_back(); +} + +} //namespace tools diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 7980b381f..35135ea18 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -31,10 +31,7 @@ #include "command_line.h" #include <boost/algorithm/string/compare.hpp> #include <boost/algorithm/string/predicate.hpp> -#include <unordered_set> #include "common/i18n.h" -#include "cryptonote_config.h" -#include "string_tools.h" namespace command_line { diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 606a2c7b7..492648afa 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -33,13 +33,11 @@ #include <stdlib.h> #include "include_base_utils.h" #include <random> -#include <boost/filesystem/fstream.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/thread.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/optional.hpp> using namespace epee; -namespace bf = boost::filesystem; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.dns" diff --git a/src/common/download.cpp b/src/common/download.cpp index 6698a5abf..58ce0595f 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -29,10 +29,7 @@ #include <string> #include <atomic> #include <boost/filesystem.hpp> -#include <boost/asio.hpp> #include <boost/thread/thread.hpp> -#include "cryptonote_config.h" -#include "include_base_utils.h" #include "file_io_utils.h" #include "net/http_client.h" #include "download.h" diff --git a/src/common/http_connection.h b/src/common/http_connection.h index 9fc6be261..554dd832b 100644 --- a/src/common/http_connection.h +++ b/src/common/http_connection.h @@ -55,7 +55,8 @@ public: { if (m_ok) { - mp_http_client->disconnect(); + try { mp_http_client->disconnect(); } + catch (...) { /* do not propagate through dtor */ } } } diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index 4a89876fb..ffe8d8b52 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -31,9 +31,7 @@ #include <ctype.h> #include <string> #include <map> -#include "include_base_utils.h" #include "file_io_utils.h" -#include "common/util.h" #include "common/i18n.h" #include "translation_files.h" diff --git a/src/common/int-util.h b/src/common/int-util.h deleted file mode 100644 index 3bcc085e2..000000000 --- a/src/common/int-util.h +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (c) 2014-2018, The Monero Project -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers - -#pragma once - -#include <assert.h> -#include <stdbool.h> -#include <stdint.h> -#include <string.h> - -#ifndef _MSC_VER -#include <sys/param.h> -#endif - -#if defined(__ANDROID__) -#include <byteswap.h> -#endif - -#if defined(__sun) && defined(__SVR4) -#include <endian.h> -#endif - -#if defined(_MSC_VER) -#include <stdlib.h> - -static inline uint32_t rol32(uint32_t x, int r) { - static_assert(sizeof(uint32_t) == sizeof(unsigned int), "this code assumes 32-bit integers"); - return _rotl(x, r); -} - -static inline uint64_t rol64(uint64_t x, int r) { - return _rotl64(x, r); -} - -#else - -static inline uint32_t rol32(uint32_t x, int r) { - return (x << (r & 31)) | (x >> (-r & 31)); -} - -static inline uint64_t rol64(uint64_t x, int r) { - return (x << (r & 63)) | (x >> (-r & 63)); -} - -#endif - -static inline uint64_t hi_dword(uint64_t val) { - return val >> 32; -} - -static inline uint64_t lo_dword(uint64_t val) { - return val & 0xFFFFFFFF; -} - -static inline uint64_t mul128(uint64_t multiplier, uint64_t multiplicand, uint64_t* product_hi) { - // multiplier = ab = a * 2^32 + b - // multiplicand = cd = c * 2^32 + d - // ab * cd = a * c * 2^64 + (a * d + b * c) * 2^32 + b * d - uint64_t a = hi_dword(multiplier); - uint64_t b = lo_dword(multiplier); - uint64_t c = hi_dword(multiplicand); - uint64_t d = lo_dword(multiplicand); - - uint64_t ac = a * c; - uint64_t ad = a * d; - uint64_t bc = b * c; - uint64_t bd = b * d; - - uint64_t adbc = ad + bc; - uint64_t adbc_carry = adbc < ad ? 1 : 0; - - // multiplier * multiplicand = product_hi * 2^64 + product_lo - uint64_t product_lo = bd + (adbc << 32); - uint64_t product_lo_carry = product_lo < bd ? 1 : 0; - *product_hi = ac + (adbc >> 32) + (adbc_carry << 32) + product_lo_carry; - assert(ac <= *product_hi); - - return product_lo; -} - -static inline uint64_t div_with_reminder(uint64_t dividend, uint32_t divisor, uint32_t* remainder) { - dividend |= ((uint64_t)*remainder) << 32; - *remainder = dividend % divisor; - return dividend / divisor; -} - -// Long division with 2^32 base -static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uint32_t divisor, uint64_t* quotient_hi, uint64_t* quotient_lo) { - uint64_t dividend_dwords[4]; - uint32_t remainder = 0; - - dividend_dwords[3] = hi_dword(dividend_hi); - dividend_dwords[2] = lo_dword(dividend_hi); - dividend_dwords[1] = hi_dword(dividend_lo); - dividend_dwords[0] = lo_dword(dividend_lo); - - *quotient_hi = div_with_reminder(dividend_dwords[3], divisor, &remainder) << 32; - *quotient_hi |= div_with_reminder(dividend_dwords[2], divisor, &remainder); - *quotient_lo = div_with_reminder(dividend_dwords[1], divisor, &remainder) << 32; - *quotient_lo |= div_with_reminder(dividend_dwords[0], divisor, &remainder); - - return remainder; -} - -#define IDENT32(x) ((uint32_t) (x)) -#define IDENT64(x) ((uint64_t) (x)) - -#define SWAP32(x) ((((uint32_t) (x) & 0x000000ff) << 24) | \ - (((uint32_t) (x) & 0x0000ff00) << 8) | \ - (((uint32_t) (x) & 0x00ff0000) >> 8) | \ - (((uint32_t) (x) & 0xff000000) >> 24)) -#define SWAP64(x) ((((uint64_t) (x) & 0x00000000000000ff) << 56) | \ - (((uint64_t) (x) & 0x000000000000ff00) << 40) | \ - (((uint64_t) (x) & 0x0000000000ff0000) << 24) | \ - (((uint64_t) (x) & 0x00000000ff000000) << 8) | \ - (((uint64_t) (x) & 0x000000ff00000000) >> 8) | \ - (((uint64_t) (x) & 0x0000ff0000000000) >> 24) | \ - (((uint64_t) (x) & 0x00ff000000000000) >> 40) | \ - (((uint64_t) (x) & 0xff00000000000000) >> 56)) - -static inline uint32_t ident32(uint32_t x) { return x; } -static inline uint64_t ident64(uint64_t x) { return x; } - -#ifndef __OpenBSD__ -# if defined(__ANDROID__) && defined(__swap32) && !defined(swap32) -# define swap32 __swap32 -# elif !defined(swap32) -static inline uint32_t swap32(uint32_t x) { - x = ((x & 0x00ff00ff) << 8) | ((x & 0xff00ff00) >> 8); - return (x << 16) | (x >> 16); -} -# endif -# if defined(__ANDROID__) && defined(__swap64) && !defined(swap64) -# define swap64 __swap64 -# elif !defined(swap64) -static inline uint64_t swap64(uint64_t x) { - x = ((x & 0x00ff00ff00ff00ff) << 8) | ((x & 0xff00ff00ff00ff00) >> 8); - x = ((x & 0x0000ffff0000ffff) << 16) | ((x & 0xffff0000ffff0000) >> 16); - return (x << 32) | (x >> 32); -} -# endif -#endif /* __OpenBSD__ */ - -#if defined(__GNUC__) -#define UNUSED __attribute__((unused)) -#else -#define UNUSED -#endif -static inline void mem_inplace_ident(void *mem UNUSED, size_t n UNUSED) { } -#undef UNUSED - -static inline void mem_inplace_swap32(void *mem, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint32_t *) mem)[i] = swap32(((const uint32_t *) mem)[i]); - } -} -static inline void mem_inplace_swap64(void *mem, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint64_t *) mem)[i] = swap64(((const uint64_t *) mem)[i]); - } -} - -static inline void memcpy_ident32(void *dst, const void *src, size_t n) { - memcpy(dst, src, 4 * n); -} -static inline void memcpy_ident64(void *dst, const void *src, size_t n) { - memcpy(dst, src, 8 * n); -} - -static inline void memcpy_swap32(void *dst, const void *src, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint32_t *) dst)[i] = swap32(((const uint32_t *) src)[i]); - } -} -static inline void memcpy_swap64(void *dst, const void *src, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint64_t *) dst)[i] = swap64(((const uint64_t *) src)[i]); - } -} - -#ifdef _MSC_VER -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# define BYTE_ORDER LITTLE_ENDIAN -#endif - -#if !defined(BYTE_ORDER) || !defined(LITTLE_ENDIAN) || !defined(BIG_ENDIAN) -static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not enabled"); -#endif - -#if BYTE_ORDER == LITTLE_ENDIAN -#define SWAP32LE IDENT32 -#define SWAP32BE SWAP32 -#define swap32le ident32 -#define swap32be swap32 -#define mem_inplace_swap32le mem_inplace_ident -#define mem_inplace_swap32be mem_inplace_swap32 -#define memcpy_swap32le memcpy_ident32 -#define memcpy_swap32be memcpy_swap32 -#define SWAP64LE IDENT64 -#define SWAP64BE SWAP64 -#define swap64le ident64 -#define swap64be swap64 -#define mem_inplace_swap64le mem_inplace_ident -#define mem_inplace_swap64be mem_inplace_swap64 -#define memcpy_swap64le memcpy_ident64 -#define memcpy_swap64be memcpy_swap64 -#endif - -#if BYTE_ORDER == BIG_ENDIAN -#define SWAP32BE IDENT32 -#define SWAP32LE SWAP32 -#define swap32be ident32 -#define swap32le swap32 -#define mem_inplace_swap32be mem_inplace_ident -#define mem_inplace_swap32le mem_inplace_swap32 -#define memcpy_swap32be memcpy_ident32 -#define memcpy_swap32le memcpy_swap32 -#define SWAP64BE IDENT64 -#define SWAP64LE SWAP64 -#define swap64be ident64 -#define swap64le swap64 -#define mem_inplace_swap64be mem_inplace_ident -#define mem_inplace_swap64le mem_inplace_swap64 -#define memcpy_swap64be memcpy_ident64 -#define memcpy_swap64le memcpy_swap64 -#endif diff --git a/src/common/password.cpp b/src/common/password.cpp index b3c51128f..5f5cb800a 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -31,7 +31,6 @@ #include "password.h" #include <iostream> -#include <memory.h> #include <stdio.h> #if defined(_WIN32) @@ -42,8 +41,6 @@ #include <unistd.h> #endif -#include "memwipe.h" - #define EOT 0x4 namespace diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index d887a13c9..42f439ad8 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -101,13 +101,13 @@ public: MCLOG_FILE(m_log_level, "msgwriter", m_oss.str()); + PAUSE_READLINE(); if (epee::console_color_default == m_color) { std::cout << m_oss.str(); } else { - PAUSE_READLINE(); set_console_color(m_color, m_bright); std::cout << m_oss.str(); epee::reset_console_color(); diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 37825e31d..cbf7163c5 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -28,10 +28,6 @@ #include "misc_log_ex.h" #include "common/threadpool.h" -#include <cassert> -#include <limits> -#include <stdexcept> - #include "cryptonote_config.h" #include "common/util.h" diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index 8a22a4b93..efdeef8d1 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -19,7 +19,7 @@ Issue Date: 20/12/2007 */ #include <stdint.h> -#include "common/int-util.h" +#include "int-util.h" #if defined(__cplusplus) extern "C" diff --git a/src/crypto/chacha.c b/src/crypto/chacha.c index 5d3edb98d..d734e8b1b 100644 --- a/src/crypto/chacha.c +++ b/src/crypto/chacha.c @@ -11,7 +11,7 @@ Public domain. #endif #include "chacha.h" -#include "common/int-util.h" +#include "int-util.h" #include "warnings.h" /* diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index ad7721cf0..ddf072f68 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -34,7 +34,6 @@ #include <cstdint> #include <cstdlib> #include <cstring> -#include <memory> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> #include <boost/shared_ptr.hpp> diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 33cc0a25a..f22df1230 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -32,14 +32,11 @@ #include <cstddef> #include <iostream> -#include <boost/thread/mutex.hpp> -#include <boost/thread/lock_guard.hpp> #include <boost/optional.hpp> #include <type_traits> #include <vector> #include "common/pod-class.h" -#include "common/util.h" #include "memwipe.h" #include "mlocker.h" #include "generic-ops.h" diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index 53594c569..12472dced 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -29,7 +29,7 @@ #ifndef __tables_h #define __tables_h -#include "common/int-util.h" +#include "int-util.h" #if BYTE_ORDER == LITTLE_ENDIAN diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index d77d55cf3..77b52e2d4 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -37,7 +37,7 @@ #include <stddef.h> #include <stdint.h> -#include "common/int-util.h" +#include "int-util.h" #include "warnings.h" static inline void *padd(void *p, size_t i) { diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index b095b5ce2..170911262 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -5,7 +5,7 @@ #include <stdio.h> #include <stdlib.h> #include <unistd.h> -#include "common/int-util.h" +#include "int-util.h" #include "hash-ops.h" #include "keccak.h" diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index a50a28e6b..8a1640e57 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -114,7 +114,7 @@ typedef uint64_t u64b_t; /* 64-bit unsigned integer */ #ifndef SKEIN_NEED_SWAP /* compile-time "override" for endianness? */ -#include "common/int-util.h" +#include "int-util.h" #define IS_BIG_ENDIAN 4321 /* byte 0 is most significant (mc68k) */ #define IS_LITTLE_ENDIAN 1234 /* byte 0 is least significant (i386) */ diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index dcbabccab..ae0bd4e98 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -35,7 +35,7 @@ #include <stdio.h> #include <unistd.h> -#include "common/int-util.h" +#include "int-util.h" #include "hash-ops.h" #include "oaes_lib.h" #include "variant2_int_sqrt.h" diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index c4e10851e..41b5f19f0 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -40,7 +40,7 @@ using namespace epee; #include "misc_language.h" #include "common/base58.h" #include "crypto/hash.h" -#include "common/int-util.h" +#include "int-util.h" #include "common/dns_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index e26aac76b..d41bc1087 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -28,9 +28,6 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#include "include_base_utils.h" -using namespace epee; - #include <atomic> #include <boost/algorithm/string.hpp> #include "wipeable_string.h" @@ -42,6 +39,8 @@ using namespace epee; #include "crypto/hash.h" #include "ringct/rctSigs.h" +using namespace epee; + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index cb2a00a12..55e3e93b3 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -34,7 +34,7 @@ #include <cstdint> #include <vector> -#include "common/int-util.h" +#include "int-util.h" #include "crypto/hash.h" #include "cryptonote_config.h" #include "difficulty.h" diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index d8ca2dd35..f4de2ed7e 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -33,8 +33,6 @@ #include <boost/utility/value_init.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/algorithm/string.hpp> -#include <boost/limits.hpp> -#include "include_base_utils.h" #include "misc_language.h" #include "syncobj.h" #include "cryptonote_basic_impl.h" @@ -54,19 +52,22 @@ #include <mach/mach_host.h> #include <AvailabilityMacros.h> #include <TargetConditionals.h> -#endif - -#ifdef __FreeBSD__ -#include <devstat.h> -#include <errno.h> -#include <fcntl.h> -#include <machine/apm_bios.h> -#include <stdio.h> -#include <sys/resource.h> -#include <sys/sysctl.h> -#include <sys/times.h> -#include <sys/types.h> -#include <unistd.h> +#elif defined(__linux__) + #include <unistd.h> + #include <sys/resource.h> + #include <sys/times.h> + #include <time.h> +#elif defined(__FreeBSD__) + #include <devstat.h> + #include <errno.h> + #include <fcntl.h> + #include <machine/apm_bios.h> + #include <stdio.h> + #include <sys/resource.h> + #include <sys/sysctl.h> + #include <sys/times.h> + #include <sys/types.h> + #include <unistd.h> #endif #undef MONERO_DEFAULT_LOG_CATEGORY @@ -146,7 +147,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------------- bool miner::request_block_template() { - block bl = AUTO_VAL_INIT(bl); + block bl; difficulty_type di = AUTO_VAL_INIT(di); uint64_t height = AUTO_VAL_INIT(height); uint64_t expected_reward; //only used for RPC calls - could possibly be useful here too? diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 2bff784c7..e16d9f3b8 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -38,11 +38,6 @@ #include "math_helper.h" #ifdef _WIN32 #include <windows.h> -#elif defined(__linux__) -#include <unistd.h> -#include <sys/resource.h> -#include <sys/times.h> -#include <time.h> #endif namespace cryptonote diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index e80e3f66c..ca569a16f 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -44,7 +44,7 @@ #include "misc_language.h" #include "profile_tools.h" #include "file_io_utils.h" -#include "common/int-util.h" +#include "int-util.h" #include "common/threadpool.h" #include "common/boost_serialization_helper.h" #include "warnings.h" @@ -229,7 +229,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke { try { - m_db->get_output_key(tx_in_to_key.amount, absolute_offsets, outputs, true); + m_db->get_output_key(epee::span<const uint64_t>(&tx_in_to_key.amount, 1), absolute_offsets, outputs, true); if (absolute_offsets.size() != outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -255,7 +255,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke add_offsets.push_back(absolute_offsets[i]); try { - m_db->get_output_key(tx_in_to_key.amount, add_offsets, add_outputs, true); + m_db->get_output_key(epee::span<const uint64_t>(&tx_in_to_key.amount, 1), add_offsets, add_outputs, true); if (add_offsets.size() != add_outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -404,7 +404,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline if(!m_db->height()) { MINFO("Blockchain not loaded, generating genesis block."); - block bl = boost::value_initialized<block>(); + block bl; block_verification_context bvc = boost::value_initialized<block_verification_context>(); generate_genesis_block(bl, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); add_new_block(bl, bvc); @@ -1767,16 +1767,34 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA res.outs.clear(); res.outs.reserve(req.outputs.size()); + + std::vector<cryptonote::output_data_t> data; try { + std::vector<uint64_t> amounts, offsets; + amounts.reserve(req.outputs.size()); + offsets.reserve(req.outputs.size()); for (const auto &i: req.outputs) { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); + amounts.push_back(i.amount); + offsets.push_back(i.index); + } + m_db->get_output_key(epee::span<const uint64_t>(amounts.data(), amounts.size()), offsets, data); + if (data.size() != req.outputs.size()) + { + MERROR("Unexpected output data size: expected " << req.outputs.size() << ", got " << data.size()); + return false; + } + for (const auto &t: data) + res.outs.push_back({t.pubkey, t.commitment, is_tx_spendtime_unlocked(t.unlock_time), t.height, crypto::null_hash}); - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + if (req.get_txid) + { + for (size_t i = 0; i < req.outputs.size(); ++i) + { + tx_out_index toi = m_db->get_output_tx_and_index(req.outputs[i].amount, req.outputs[i].index); + res.outs[i].txid = toi.first; + } } } catch (const std::exception &e) @@ -3795,7 +3813,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin { try { - m_db->get_output_key(amount, offsets, outputs, true); + m_db->get_output_key(epee::span<const uint64_t>(&amount, 1), offsets, outputs, true); } catch (const std::exception& e) { diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 10ab3fe65..b353cc5d1 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -30,13 +30,11 @@ #include <boost/algorithm/string.hpp> -#include "include_base_utils.h" #include "string_tools.h" using namespace epee; #include <unordered_set> #include "cryptonote_core.h" -#include "common/command_line.h" #include "common/util.h" #include "common/updates.h" #include "common/download.h" @@ -45,7 +43,6 @@ using namespace epee; #include "warnings.h" #include "crypto/crypto.h" #include "cryptonote_config.h" -#include "cryptonote_tx_utils.h" #include "misc_language.h" #include "file_io_utils.h" #include <csignal> diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 2eb6c842b..46c3c3e8b 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -34,7 +34,6 @@ #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> -#include <boost/interprocess/sync/file_lock.hpp> #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index e2900916b..d4b4e4d34 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -40,7 +40,7 @@ #include "blockchain.h" #include "blockchain_db/blockchain_db.h" #include "common/boost_serialization_helper.h" -#include "common/int-util.h" +#include "int-util.h" #include "misc_language.h" #include "warnings.h" #include "common/perf_timer.h" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index c9fd40d88..6d9ad9028 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -30,20 +30,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include <boost/asio.hpp> #include <string> #include <vector> -#include <boost/noncopyable.hpp> -#include <boost/shared_ptr.hpp> -#include <atomic> - -#include <boost/asio.hpp> -#include <boost/array.hpp> -#include <boost/noncopyable.hpp> -#include <boost/shared_ptr.hpp> -#include <boost/enable_shared_from_this.hpp> -#include <boost/interprocess/detail/atomic.hpp> -#include <boost/thread/thread.hpp> #include <memory> @@ -51,24 +39,14 @@ #include "net/net_utils_base.h" #include "misc_log_ex.h" -#include <boost/lambda/bind.hpp> -#include <boost/lambda/lambda.hpp> -#include <boost/uuid/random_generator.hpp> #include <boost/chrono.hpp> -#include <boost/utility/value_init.hpp> -#include <boost/asio/deadline_timer.hpp> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread/thread.hpp> #include "misc_language.h" #include "pragma_comp_defs.h" -#include <sstream> -#include <iomanip> #include <algorithm> -#include <boost/asio/basic_socket.hpp> -#include <boost/asio/ip/unicast.hpp> - #include "cryptonote_protocol_handler.h" #include "net/network_throttle.hpp" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index c2c660e8c..6081ddd89 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1077,8 +1077,10 @@ skip: if(tvc[i].m_verifivation_failed) { if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{ + cryptonote::transaction tx; + parse_and_validate_tx_from_blob(*it, tx); // must succeed if we got here LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, tx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(*it)) << ", dropping connection"); + << epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(tx)) << ", dropping connection"); drop_connection(context, false, true); return 1; })) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49d6d49cf..13478f39d 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -75,18 +75,15 @@ public: protocol.set_p2p_endpoint(p2p.get()); core.set_protocol(protocol.get()); - const auto testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - const auto stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - const auto regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); const auto restricted = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc); const auto main_rpc_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); - rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : regtest ? cryptonote::FAKECHAIN : cryptonote::MAINNET, main_rpc_port, "core"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, main_rpc_port, "core"}); auto restricted_rpc_port_arg = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; if(!command_line::is_arg_defaulted(vm, restricted_rpc_port_arg)) { auto restricted_rpc_port = command_line::get_arg(vm, restricted_rpc_port_arg); - rpcs.emplace_back(new t_rpc{vm, core, p2p, true, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET, restricted_rpc_port, "restricted"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, true, restricted_rpc_port, "restricted"}); } } }; diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index 9621b0d01..37dffc097 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -54,7 +54,6 @@ public: , t_core & core , t_p2p & p2p , const bool restricted - , const cryptonote::network_type nettype , const std::string & port , const std::string & description ) @@ -62,7 +61,7 @@ public: { MGINFO("Initializing " << m_description << " RPC server..."); - if (!m_server.init(vm, restricted, nettype, port)) + if (!m_server.init(vm, restricted, port)) { throw std::runtime_error("Failed to initialize " + m_description + " RPC server."); } diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp index 3cbee9c51..5af4e1a4a 100644 --- a/src/daemonizer/posix_fork.cpp +++ b/src/daemonizer/posix_fork.cpp @@ -12,7 +12,6 @@ #include <unistd.h> #include <stdexcept> #include <string> -#include <sys/stat.h> #ifndef TMPDIR #define TMPDIR "/tmp" diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 1e3d80949..2286998a4 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -31,7 +31,7 @@ #include "device_default.hpp" -#include "common/int-util.h" +#include "int-util.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" #include "ringct/rctOps.h" diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt index c555e9fcd..b27c843b6 100644 --- a/src/device_trezor/CMakeLists.txt +++ b/src/device_trezor/CMakeLists.txt @@ -63,37 +63,20 @@ set(trezor_sources set(trezor_private_headers) -include(FindProtobuf) -find_package(Protobuf) # REQUIRED - -# Test for HAVE_PROTOBUF from the parent -if(Protobuf_FOUND AND HAVE_PROTOBUF) - if ("$ENV{PYTHON3}" STREQUAL "") - set(PYTHON3 "python3") - else() - set(PYTHON3 "$ENV{PYTHON3}" CACHE INTERNAL "Copied from environment variable") - endif() - - execute_process(COMMAND ${PYTHON3} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) - if(RET) - message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})." - "OUT: ${OUT}, ERR: ${ERR}." - "Please read src/device_trezor/trezor/tools/README.md") - else() - message(STATUS "Trezor protobuf messages regenerated ${OUT}") - set(TREZOR_PROTOBUF_GENERATED 1) - endif() -endif() - - -if(TREZOR_PROTOBUF_GENERATED) +# Protobuf and LibUSB processed by CheckTrezor +if(DEVICE_TREZOR_READY) message(STATUS "Trezor support enabled") add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) + set(TREZOR_LIBUSB_LIBRARIES "") + if(LibUSB_COMPILE_TEST_PASSED) + list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES}) + message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") + endif() + monero_private_headers(device_trezor - ${device_private_headers} - ${PROTOBUF_INCLUDE_DIR}) + ${device_private_headers}) monero_add_library(device_trezor ${trezor_sources} @@ -109,14 +92,13 @@ if(TREZOR_PROTOBUF_GENERATED) common ${SODIUM_LIBRARY} ${Boost_CHRONO_LIBRARY} - ${PROTOBUF_LIBRARY} + ${Protobuf_LIBRARY} + ${TREZOR_LIBUSB_LIBRARIES} PRIVATE ${EXTRA_LIBRARIES}) - # set(WITH_DEVICE_TREZOR 1 PARENT_SCOPE) - # add_definitions(-DWITH_DEVICE_TREZOR=1) - else() + message(STATUS "Trezor support disabled") monero_private_headers(device_trezor) monero_add_library(device_trezor device_trezor.cpp) target_link_libraries(device_trezor PUBLIC cncrypto) diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index f55cbb15d..5096fcea8 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -32,13 +32,12 @@ namespace hw { namespace trezor { -#if WITH_DEVICE_TREZOR +#ifdef WITH_DEVICE_TREZOR #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" #define HW_TREZOR_NAME "Trezor" -#define HW_TREZOR_NAME_LITE "TrezorLite" static device_trezor *trezor_device = nullptr; static device_trezor *ensure_trezor_device(){ diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 765c9b82c..a2134574c 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -49,7 +49,7 @@ namespace trezor { void register_all(); void register_all(std::map<std::string, std::unique_ptr<device>> ®istry); -#if WITH_DEVICE_TREZOR +#ifdef WITH_DEVICE_TREZOR class device_trezor; /** diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 3a98bba5a..38c20c30b 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -32,43 +32,11 @@ namespace hw { namespace trezor { -#if WITH_DEVICE_TREZOR +#ifdef WITH_DEVICE_TREZOR #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" - std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_button_request(const messages::common::ButtonRequest * msg){ - MDEBUG("on_button_request"); - device.on_button_request(); - return std::make_shared<messages::common::ButtonAck>(); - } - - std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_pin_matrix_request(const messages::common::PinMatrixRequest * msg){ - MDEBUG("on_pin_request"); - epee::wipeable_string pin; - device.on_pin_request(pin); - auto resp = std::make_shared<messages::common::PinMatrixAck>(); - resp->set_pin(pin.data(), pin.size()); - return resp; - } - - std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_passphrase_request(const messages::common::PassphraseRequest * msg){ - MDEBUG("on_passhprase_request"); - epee::wipeable_string passphrase; - device.on_passphrase_request(msg->on_device(), passphrase); - auto resp = std::make_shared<messages::common::PassphraseAck>(); - if (!msg->on_device()){ - resp->set_passphrase(passphrase.data(), passphrase.size()); - } - return resp; - } - - std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_passphrase_state_request(const messages::common::PassphraseStateRequest * msg){ - MDEBUG("on_passhprase_state_request"); - device.on_passphrase_state_request(msg->state()); - return std::make_shared<messages::common::PassphraseStateAck>(); - } - const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080, 0x80000000}; device_trezor_base::device_trezor_base() { @@ -117,9 +85,6 @@ namespace trezor { return false; } - if (!m_protocol_callback){ - m_protocol_callback = std::make_shared<trezor_protocol_callback>(*this); - } return true; } @@ -245,6 +210,49 @@ namespace trezor { } } + void device_trezor_base::write_raw(const google::protobuf::Message * msg){ + require_connected(); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + this->getTransport()->write(*msg); + } + + GenericMessage device_trezor_base::read_raw(){ + require_connected(); + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + + this->getTransport()->read(msg_resp, &msg_resp_type); + return GenericMessage(msg_resp_type, msg_resp); + } + + GenericMessage device_trezor_base::call_raw(const google::protobuf::Message * msg) { + write_raw(msg); + return read_raw(); + } + + bool device_trezor_base::message_handler(GenericMessage & input){ + // Later if needed this generic message handler can be replaced by a pointer to + // a protocol message handler which by default points to the device class which implements + // the default handler. + switch(input.m_type){ + case messages::MessageType_ButtonRequest: + on_button_request(input, dynamic_cast<const messages::common::ButtonRequest*>(input.m_msg.get())); + return true; + case messages::MessageType_PassphraseRequest: + on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get())); + return true; + case messages::MessageType_PassphraseStateRequest: + on_passphrase_state_request(input, dynamic_cast<const messages::common::PassphraseStateRequest*>(input.m_msg.get())); + return true; + case messages::MessageType_PinMatrixRequest: + on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get())); + return true; + default: + return false; + } + } + + /* ======================================================================= */ /* TREZOR PROTOCOL */ /* ======================================================================= */ @@ -269,32 +277,67 @@ namespace trezor { return false; } - void device_trezor_base::on_button_request() + void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) { + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + MDEBUG("on_button_request, code: " << msg->code()); + + messages::common::ButtonAck ack; + write_raw(&ack); + if (m_callback){ m_callback->on_button_request(); } + + resp = read_raw(); } - void device_trezor_base::on_pin_request(epee::wipeable_string & pin) + void device_trezor_base::on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg) { + MDEBUG("on_pin_request"); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + + epee::wipeable_string pin; + if (m_callback){ m_callback->on_pin_request(pin); } + + // TODO: remove PIN from memory + messages::common::PinMatrixAck m; + m.set_pin(pin.data(), pin.size()); + resp = call_raw(&m); } - void device_trezor_base::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) + void device_trezor_base::on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg) { + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + MDEBUG("on_passhprase_request, on device: " << msg->on_device()); + epee::wipeable_string passphrase; + if (m_callback){ - m_callback->on_passphrase_request(on_device, passphrase); + m_callback->on_passphrase_request(msg->on_device(), passphrase); } + + messages::common::PassphraseAck m; + if (!msg->on_device()){ + // TODO: remove passphrase from memory + m.set_passphrase(passphrase.data(), passphrase.size()); + } + resp = call_raw(&m); } - void device_trezor_base::on_passphrase_state_request(const std::string & state) + void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg) { + MDEBUG("on_passhprase_state_request"); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + if (m_callback){ - m_callback->on_passphrase_state_request(state); + m_callback->on_passphrase_state_request(msg->state()); } + + messages::common::PassphraseStateAck m; + resp = call_raw(&m); } #endif //WITH_DEVICE_TREZOR diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 644a49332..88d419494 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -54,7 +54,7 @@ namespace hw { namespace trezor { -#if WITH_DEVICE_TREZOR +#ifdef WITH_DEVICE_TREZOR class device_trezor_base; /** @@ -69,41 +69,6 @@ namespace trezor { }; /** - * Default Trezor protocol client callback - */ - class trezor_protocol_callback { - protected: - device_trezor_base & device; - - public: - explicit trezor_protocol_callback(device_trezor_base & device): device(device) {} - - std::shared_ptr<google::protobuf::Message> on_button_request(const messages::common::ButtonRequest * msg); - std::shared_ptr<google::protobuf::Message> on_pin_matrix_request(const messages::common::PinMatrixRequest * msg); - std::shared_ptr<google::protobuf::Message> on_passphrase_request(const messages::common::PassphraseRequest * msg); - std::shared_ptr<google::protobuf::Message> on_passphrase_state_request(const messages::common::PassphraseStateRequest * msg); - - std::shared_ptr<google::protobuf::Message> on_message(const google::protobuf::Message * msg, messages::MessageType message_type){ - MDEBUG("on_general_message"); - return on_message_dispatch(msg, message_type); - } - - std::shared_ptr<google::protobuf::Message> on_message_dispatch(const google::protobuf::Message * msg, messages::MessageType message_type){ - if (message_type == messages::MessageType_ButtonRequest){ - return on_button_request(dynamic_cast<const messages::common::ButtonRequest*>(msg)); - } else if (message_type == messages::MessageType_PassphraseRequest) { - return on_passphrase_request(dynamic_cast<const messages::common::PassphraseRequest*>(msg)); - } else if (message_type == messages::MessageType_PassphraseStateRequest) { - return on_passphrase_state_request(dynamic_cast<const messages::common::PassphraseStateRequest*>(msg)); - } else if (message_type == messages::MessageType_PinMatrixRequest) { - return on_pin_matrix_request(dynamic_cast<const messages::common::PinMatrixRequest*>(msg)); - } else { - return nullptr; - } - } - }; - - /** * TREZOR device template with basic functions */ class device_trezor_base : public hw::core::device_default { @@ -114,7 +79,6 @@ namespace trezor { mutable boost::mutex command_locker; std::shared_ptr<Transport> m_transport; - std::shared_ptr<trezor_protocol_callback> m_protocol_callback; std::shared_ptr<trezor_callback> m_callback; std::string full_name; @@ -129,6 +93,15 @@ namespace trezor { void call_ping_unsafe(); void test_ping(); + // Communication methods + + void write_raw(const google::protobuf::Message * msg); + GenericMessage read_raw(); + GenericMessage call_raw(const google::protobuf::Message * msg); + + // Trezor message protocol handler. Handles specific signalling messages. + bool message_handler(GenericMessage & input); + /** * Client communication wrapper, handles specific Trezor protocol. * @@ -141,8 +114,7 @@ namespace trezor { const boost::optional<messages::MessageType> & resp_type = boost::none, const boost::optional<std::vector<messages::MessageType>> & resp_types = boost::none, const boost::optional<messages::MessageType*> & resp_type_ptr = boost::none, - bool open_session = false, - unsigned depth=0) + bool open_session = false) { // Require strictly protocol buffers response in the template. BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); @@ -151,8 +123,12 @@ namespace trezor { throw std::invalid_argument("Cannot specify list of accepted types and not using generic response"); } + // Determine type of expected message response + const messages::MessageType required_type = accepting_base ? messages::MessageType_Success : + (resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>()); + // Open session if required - if (open_session && depth == 0){ + if (open_session){ try { m_transport->open(); } catch (const std::exception& e) { @@ -162,47 +138,37 @@ namespace trezor { // Scoped session closer BOOST_SCOPE_EXIT_ALL(&, this) { - if (open_session && depth == 0){ + if (open_session){ this->getTransport()->close(); } }; - // Write the request + // Write/read the request CHECK_AND_ASSERT_THROW_MES(req, "Request is null"); - this->getTransport()->write(*req); + auto msg_resp = call_raw(req.get()); - // Read the response - std::shared_ptr<google::protobuf::Message> msg_resp; - hw::trezor::messages::MessageType msg_resp_type; + bool processed = false; + do { + processed = message_handler(msg_resp); + } while(processed); - // We may have several roundtrips with the handler - this->getTransport()->read(msg_resp, &msg_resp_type); + // Response section if (resp_type_ptr){ - *(resp_type_ptr.get()) = msg_resp_type; + *(resp_type_ptr.get()) = msg_resp.m_type; } - // Determine type of expected message response - messages::MessageType required_type = accepting_base ? messages::MessageType_Success : - (resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>()); + if (msg_resp.m_type == messages::MessageType_Failure) { + throw_failure_exception(dynamic_cast<messages::common::Failure *>(msg_resp.m_msg.get())); - if (msg_resp_type == messages::MessageType_Failure) { - throw_failure_exception(dynamic_cast<messages::common::Failure *>(msg_resp.get())); + } else if (!accepting_base && msg_resp.m_type == required_type) { + return message_ptr_retype<t_message>(msg_resp.m_msg); - } else if (!accepting_base && msg_resp_type == required_type) { - return message_ptr_retype<t_message>(msg_resp); + } else if (accepting_base && (!resp_types || + std::find(resp_types.get().begin(), resp_types.get().end(), msg_resp.m_type) != resp_types.get().end())) { + return message_ptr_retype<t_message>(msg_resp.m_msg); } else { - auto resp = this->getProtocolCallback()->on_message(msg_resp.get(), msg_resp_type); - if (resp) { - return this->client_exchange<t_message>(resp, boost::none, resp_types, resp_type_ptr, false, depth + 1); - - } else if (accepting_base && (!resp_types || - std::find(resp_types.get().begin(), resp_types.get().end(), msg_resp_type) != resp_types.get().end())) { - return message_ptr_retype<t_message>(msg_resp); - - } else { - throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); - } + throw exc::UnexpectedMessageException(msg_resp.m_type, msg_resp.m_msg); } } @@ -252,10 +218,6 @@ namespace trezor { return m_transport; } - std::shared_ptr<trezor_protocol_callback> getProtocolCallback(){ - return m_protocol_callback; - } - std::shared_ptr<trezor_callback> getCallback(){ return m_callback; } @@ -288,10 +250,10 @@ namespace trezor { bool ping(); // Protocol callbacks - void on_button_request(); - void on_pin_request(epee::wipeable_string & pin); - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); - void on_passphrase_state_request(const std::string & state); + void on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg); + void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); + void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg); + void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg); }; #endif diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp index 8abdd2c18..97dc0a957 100644 --- a/src/device_trezor/trezor.hpp +++ b/src/device_trezor/trezor.hpp @@ -32,7 +32,7 @@ #include "trezor/trezor_defs.hpp" -#ifdef HAVE_PROTOBUF +#ifdef WITH_DEVICE_TREZOR #include "trezor/transport.hpp" #include "trezor/messages/messages.pb.h" #include "trezor/messages/messages-common.pb.h" diff --git a/src/device_trezor/trezor/tools/README.md b/src/device_trezor/trezor/tools/README.md index 91a8fb3f0..b176017ac 100644 --- a/src/device_trezor/trezor/tools/README.md +++ b/src/device_trezor/trezor/tools/README.md @@ -2,33 +2,28 @@ ## Messages rebuild -Install `protoc` for your distribution. +Install `protoc` for your distribution. Requirements: - `protobuf-compiler` - `libprotobuf-dev` -- `libprotoc-dev` -- `python-protobuf` +- `python` -Python 3 is required. If you don't have python 3 quite an easy way is -to use [pyenv]. -It is also advised to create own python virtual environment so dependencies -are installed in this project-related virtual environment. +Soft requirement: Python 3, can be easily installed with [pyenv]. -```bash -python -m venv / -``` +### Python 2 -Make sure your python has `protobuf` package installed +Workaround if there is no Python3 available: ```bash -pip install protobuf +pip install backports.tempfile ``` -Regenerate messages: +### Regenerate messages -``` -./venv/bin/python3 src/device_trezor/trezor/tools/build_protob.py +```bash +cd src/device_trezor/trezor +python tools/build_protob.py ``` The messages regeneration is done also automatically via cmake. diff --git a/src/device_trezor/trezor/tools/pb2cpp.py b/src/device_trezor/trezor/tools/pb2cpp.py index eaa8a90ed..4d7cc775f 100644 --- a/src/device_trezor/trezor/tools/pb2cpp.py +++ b/src/device_trezor/trezor/tools/pb2cpp.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Converts Google's protobuf python definitions of TREZOR wire messages # to plain-python objects as used in TREZOR Core and python-trezor @@ -8,11 +8,19 @@ import os import re import shutil import subprocess -import sys import glob -import tempfile import hashlib +try: + from tempfile import TemporaryDirectory +except: + # Py2 backward compatibility, optionally installed by user + # pip install backports.tempfile + try: + from backports.tempfile import TemporaryDirectory + except: + raise EnvironmentError('TemporaryDirectory could not be imported. Try: pip install backports.tempfile') + AUTO_HEADER = "# Automatically generated by pb2cpp\n" @@ -23,6 +31,9 @@ UNDEF_STATEMENT = """ #endif """ +PROTOC = None +PROTOC_INCLUDE = None + def which(pgm): path = os.getenv('PATH') @@ -32,15 +43,6 @@ def which(pgm): return p -PROTOC = which("protoc") -if not PROTOC: - print("protoc command not found") - sys.exit(1) - -PROTOC_PREFIX = os.path.dirname(os.path.dirname(PROTOC)) -PROTOC_INCLUDE = os.path.join(PROTOC_PREFIX, "include") - - def namespace_file(fpath, package): """Adds / replaces package name. Simple regex parsing, may use https://github.com/ph4r05/plyprotobuf later""" with open(fpath) as fh: @@ -82,9 +84,10 @@ def protoc(files, out_dir, additional_includes=(), package=None, force=False): include_dirs = set() include_dirs.add(PROTOC_INCLUDE) - include_dirs.update(additional_includes) + if additional_includes: + include_dirs.update(additional_includes) - with tempfile.TemporaryDirectory() as tmpdir_protob, tempfile.TemporaryDirectory() as tmpdir_out: + with TemporaryDirectory() as tmpdir_protob, TemporaryDirectory() as tmpdir_out: include_dirs.add(tmpdir_protob) new_files = [] @@ -125,9 +128,9 @@ def update_message_files(tmpdir_out, out_dir, force=False): dest_file = os.path.join(out_dir, bname) if not force and os.path.exists(dest_file): data = open(fname, 'rb').read() - data_hash = hashlib.sha3_256(data).digest() + data_hash = hashlib.sha256(data).digest() data_dest = open(dest_file, 'rb').read() - data_dest_hash = hashlib.sha3_256(data_dest).digest() + data_dest_hash = hashlib.sha256(data_dest).digest() if data_hash == data_dest_hash: continue @@ -163,7 +166,8 @@ def strip_leader(s, prefix): return s -if __name__ == "__main__": +def main(): + global PROTOC, PROTOC_INCLUDE logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser() @@ -179,8 +183,31 @@ if __name__ == "__main__": protoc_includes = args.protoc_include or (os.environ.get("PROTOC_INCLUDE"),) + PROTOBUF_INCLUDE_DIRS = os.getenv("PROTOBUF_INCLUDE_DIRS", None) + PROTOBUF_PROTOC_EXECUTABLE = os.getenv("PROTOBUF_PROTOC_EXECUTABLE", None) + + if PROTOBUF_PROTOC_EXECUTABLE and not os.path.exists(PROTOBUF_PROTOC_EXECUTABLE): + raise ValueError("PROTOBUF_PROTOC_EXECUTABLE set but not found: %s" % PROTOBUF_PROTOC_EXECUTABLE) + + elif PROTOBUF_PROTOC_EXECUTABLE: + PROTOC = PROTOBUF_PROTOC_EXECUTABLE + + else: + if os.name == "nt": + PROTOC = which("protoc.exe") + else: + PROTOC = which("protoc") + + if not PROTOC: + raise ValueError("protoc command not found. Set PROTOBUF_PROTOC_EXECUTABLE env var to the protoc binary and optionally PROTOBUF_INCLUDE_DIRS") + + PROTOC_PREFIX = os.path.dirname(os.path.dirname(PROTOC)) + PROTOC_INCLUDE = PROTOBUF_INCLUDE_DIRS if PROTOBUF_INCLUDE_DIRS else os.path.join(PROTOC_PREFIX, "include") + protoc( args.proto, args.out_dir, protoc_includes, package=args.namespace, force=args.force ) +if __name__ == "__main__": + main() diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index fc86177e1..814537eb6 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -27,13 +27,21 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +#ifdef WITH_DEVICE_TREZOR_WEBUSB +#include <libusb.h> +#endif + #include <boost/endian/conversion.hpp> #include <boost/asio/io_service.hpp> #include <boost/asio/ip/udp.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> +#include <boost/format.hpp> #include "transport.hpp" #include "messages/messages-common.pb.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor.transport" + using namespace std; using json = rapidjson::Document; @@ -581,12 +589,400 @@ namespace trezor{ << ">"; } +#ifdef WITH_DEVICE_TREZOR_WEBUSB + + static bool is_trezor1(libusb_device_descriptor * info){ + return info->idVendor == 0x534C && info->idProduct == 0x0001; + } + + static bool is_trezor2(libusb_device_descriptor * info){ + return info->idVendor == 0x1209 && info->idProduct == 0x53C1; + } + + static bool is_trezor2_bl(libusb_device_descriptor * info){ + return info->idVendor == 0x1209 && info->idProduct == 0x53C0; + } + + static uint8_t get_trezor_dev_mask(libusb_device_descriptor * info){ + uint8_t mask = 0; + CHECK_AND_ASSERT_THROW_MES(info, "Empty device descriptor"); + mask |= is_trezor1(info) ? 1 : 0; + mask |= is_trezor2(info) ? 2 : 0; + mask |= is_trezor2_bl(info) ? 4 : 0; + return mask; + } + + static void set_libusb_log(libusb_context *ctx){ + CHECK_AND_ASSERT_THROW_MES(ctx, "Null libusb context"); + + // http://libusb.sourceforge.net/api-1.0/group__libusb__lib.html +#if defined(LIBUSB_API_VERSION) && (LIBUSB_API_VERSION >= 0x01000106) +# define TREZOR_LIBUSB_SET_DEBUG(ctx, level) libusb_set_option(ctx, LIBUSB_OPTION_LOG_LEVEL, level) +#else +# define TREZOR_LIBUSB_SET_DEBUG(ctx, level) libusb_set_debug(ctx, level) +#endif + + if (ELPP->vRegistry()->allowed(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 3); + else if (ELPP->vRegistry()->allowed(el::Level::Warning, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 2); + else if (ELPP->vRegistry()->allowed(el::Level::Error, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 1); + +#undef TREZOR_LIBUSB_SET_DEBUG + } + + static int get_libusb_ports(libusb_device *dev, std::vector<uint8_t> &path){ + uint8_t tmp_path[16]; + int r = libusb_get_port_numbers(dev, tmp_path, sizeof(tmp_path)); + CHECK_AND_ASSERT_MES(r != LIBUSB_ERROR_OVERFLOW, -1, "Libusb path array too small"); + CHECK_AND_ASSERT_MES(r >= 0, -1, "Libusb path array error"); + + path.resize(r); + for (int i = 0; i < r; i++){ + path[i] = tmp_path[i]; + } + + return 0; + } + + static std::string get_usb_path(uint8_t bus_id, const std::vector<uint8_t> &path){ + std::stringstream ss; + ss << WebUsbTransport::PATH_PREFIX << (boost::format("%03d") % ((int)bus_id)); + for(uint8_t port : path){ + ss << ":" << ((int) port); + } + return ss.str(); + } + + const char * WebUsbTransport::PATH_PREFIX = "webusb:"; + + WebUsbTransport::WebUsbTransport( + boost::optional<libusb_device_descriptor*> descriptor, + boost::optional<std::shared_ptr<Protocol>> proto + ): m_conn_count(0), + m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr), + m_bus_id(-1), m_device_addr(-1) + { + if (descriptor){ + libusb_device_descriptor * desc = new libusb_device_descriptor; + memcpy(desc, descriptor.get(), sizeof(libusb_device_descriptor)); + this->m_usb_device_desc.reset(desc); + } + + m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>(); + +#ifdef WITH_TREZOR_DEBUG + m_debug_mode = false; +#endif + } + + WebUsbTransport::~WebUsbTransport(){ + if (m_usb_device){ + close(); + } + + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; + } + } + + void WebUsbTransport::require_device() const{ + if (!m_usb_device_desc){ + throw std::runtime_error("No USB device specified"); + } + } + + void WebUsbTransport::require_connected() const{ + require_device(); + if (!m_usb_device_handle){ + throw std::runtime_error("USB Device not opened"); + } + } + + void WebUsbTransport::enumerate(t_transport_vect & res) { + int r; + libusb_device **devs; + libusb_context *ctx = nullptr; + + r = libusb_init(&ctx); + CHECK_AND_ASSERT_THROW_MES(r >= 0, "Unable to init libusb"); + + set_libusb_log(ctx); + + ssize_t cnt = libusb_get_device_list(ctx, &devs); + if (cnt < 0){ + libusb_exit(ctx); + throw std::runtime_error("Unable to enumerate libusb devices"); + } + + MTRACE("Libusb devices: " << cnt); + + for(ssize_t i = 0; i < cnt; i++) { + libusb_device_descriptor desc{}; + r = libusb_get_device_descriptor(devs[i], &desc); + if (r < 0){ + MERROR("Unable to get libusb device descriptor " << i); + continue; + } + + const auto trezor_mask = get_trezor_dev_mask(&desc); + if (!trezor_mask){ + continue; + } + + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct << " mask " << (int)trezor_mask); + + auto t = std::make_shared<WebUsbTransport>(boost::make_optional(&desc)); + t->m_bus_id = libusb_get_bus_number(devs[i]); + t->m_device_addr = libusb_get_device_address(devs[i]); + + // Port resolution may fail. Non-critical error, just addressing precision is decreased. + get_libusb_ports(devs[i], t->m_port_numbers); + + res.push_back(t); + } + + libusb_free_device_list(devs, 1); + libusb_exit(ctx); + } + + std::string WebUsbTransport::get_path() const { + if (!m_usb_device_desc){ + return ""; + } + + return get_usb_path(static_cast<uint8_t>(m_bus_id), m_port_numbers); + }; + + void WebUsbTransport::open() { + const int interface = get_interface(); + if (m_conn_count > 0){ + MTRACE("Already opened, count: " << m_conn_count); + m_conn_count += 1; + return; + } + +#define TREZOR_DESTROY_SESSION() do { libusb_exit(m_usb_session); m_usb_session = nullptr; } while(0) + + int r; + libusb_device **devs = nullptr; + + if (m_usb_session) { + TREZOR_DESTROY_SESSION(); + } + + r = libusb_init(&m_usb_session); + CHECK_AND_ASSERT_THROW_MES(r >= 0, "Unable to init libusb"); + set_libusb_log(m_usb_session); + + bool found = false; + int open_res = 0; + + ssize_t cnt = libusb_get_device_list(m_usb_session, &devs); + if (cnt < 0){ + TREZOR_DESTROY_SESSION(); + throw std::runtime_error("Unable to enumerate libusb devices"); + } + + for (ssize_t i = 0; i < cnt; i++) { + libusb_device_descriptor desc{}; + r = libusb_get_device_descriptor(devs[i], &desc); + if (r < 0){ + MERROR("Unable to get libusb device descriptor " << i); + continue; + } + + const auto trezor_mask = get_trezor_dev_mask(&desc); + if (!trezor_mask) { + continue; + } + + auto bus_id = libusb_get_bus_number(devs[i]); + std::vector<uint8_t> path; + + // Port resolution may fail. Non-critical error, just addressing precision is decreased. + get_libusb_ports(devs[i], path); + + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct + << ", mask: " << (int)trezor_mask + << ". path: " << get_usb_path(bus_id, path)); + + if (bus_id == m_bus_id && path == m_port_numbers) { + found = true; + m_usb_device = devs[i]; + open_res = libusb_open(m_usb_device, &m_usb_device_handle); + break; + } + } + + libusb_free_device_list(devs, 1); + + if (!found){ + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Device not found"); + + } else if (found && open_res != 0) { + m_usb_device_handle = nullptr; + m_usb_device = nullptr; + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Unable to open libusb device"); + } + + r = libusb_claim_interface(m_usb_device_handle, interface); + + if (r != 0){ + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + m_usb_device = nullptr; + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Unable to claim libusb device"); + } + + m_conn_count += 1; + m_proto->session_begin(*this); + +#undef TREZOR_DESTROY_SESSION + }; + + void WebUsbTransport::close() { + m_conn_count -= 1; + + if (m_conn_count < 0){ + MERROR("Close counter is negative: " << m_conn_count); + + } else if (m_conn_count == 0){ + MTRACE("Closing webusb device"); + + m_proto->session_end(*this); + + int r = libusb_release_interface(m_usb_device_handle, get_interface()); + if (r != 0){ + MERROR("Could not release libusb interface: " << r); + } + + m_usb_device = nullptr; + if (m_usb_device_handle) { + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + } + + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; + } + } + }; + + + int WebUsbTransport::get_interface() const{ + const int INTERFACE_NORMAL = 0; +#ifdef WITH_TREZOR_DEBUG + const int INTERFACE_DEBUG = 1; + return m_debug_mode ? INTERFACE_DEBUG : INTERFACE_NORMAL; +#else + return INTERFACE_NORMAL; +#endif + } + + unsigned char WebUsbTransport::get_endpoint() const{ + const unsigned char ENDPOINT_NORMAL = 1; +#ifdef WITH_TREZOR_DEBUG + const unsigned char ENDPOINT_DEBUG = 2; + return m_debug_mode ? ENDPOINT_DEBUG : ENDPOINT_NORMAL; +#else + return ENDPOINT_NORMAL; +#endif + } + + void WebUsbTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + }; + + void WebUsbTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + }; + + void WebUsbTransport::write_chunk(const void * buff, size_t size) { + require_connected(); + if (size != REPLEN){ + throw exc::CommunicationException("Invalid chunk size: "); + } + + unsigned char endpoint = get_endpoint(); + endpoint = (endpoint & ~LIBUSB_ENDPOINT_DIR_MASK) | LIBUSB_ENDPOINT_OUT; + + int transferred = 0; + int r = libusb_interrupt_transfer(m_usb_device_handle, endpoint, (unsigned char*)buff, (int)size, &transferred, 0); + CHECK_AND_ASSERT_THROW_MES(r == 0, "Unable to transfer, r: " << r); + if (transferred != (int)size){ + throw exc::CommunicationException("Could not transfer chunk"); + } + }; + + size_t WebUsbTransport::read_chunk(void * buff, size_t size) { + require_connected(); + unsigned char endpoint = get_endpoint(); + endpoint = (endpoint & ~LIBUSB_ENDPOINT_DIR_MASK) | LIBUSB_ENDPOINT_IN; + + int transferred = 0; + int r = libusb_interrupt_transfer(m_usb_device_handle, endpoint, (unsigned char*)buff, (int)size, &transferred, 0); + CHECK_AND_ASSERT_THROW_MES(r == 0, "Unable to transfer, r: " << r); + if (transferred != (int)size){ + throw exc::CommunicationException("Could not read the chunk"); + } + + return transferred; + }; + + std::ostream& WebUsbTransport::dump(std::ostream& o) const { + o << "WebUsbTransport<path=" << get_path() + << ", vendorId=" << (m_usb_device_desc ? std::to_string(m_usb_device_desc->idVendor) : "?") + << ", productId=" << (m_usb_device_desc ? std::to_string(m_usb_device_desc->idProduct) : "?") + << ", deviceType="; + + if (m_usb_device_desc){ + if (is_trezor1(m_usb_device_desc.get())) + o << "TrezorOne"; + else if (is_trezor2(m_usb_device_desc.get())) + o << "TrezorT"; + else if (is_trezor2_bl(m_usb_device_desc.get())) + o << "TrezorT BL"; + } else { + o << "?"; + } + + return o << ">"; + }; + +#endif // WITH_DEVICE_TREZOR_WEBUSB + void enumerate(t_transport_vect & res){ BridgeTransport bt; - bt.enumerate(res); + try{ + bt.enumerate(res); + } catch (const std::exception & e){ + MERROR("BridgeTransport enumeration failed:" << e.what()); + } + +#ifdef WITH_DEVICE_TREZOR_WEBUSB + hw::trezor::WebUsbTransport btw; + try{ + btw.enumerate(res); + } catch (const std::exception & e){ + MERROR("WebUsbTransport enumeration failed:" << e.what()); + } +#endif +#ifdef WITH_DEVICE_TREZOR_UDP hw::trezor::UdpTransport btu; - btu.enumerate(res); + try{ + btu.enumerate(res); + } catch (const std::exception & e){ + MERROR("UdpTransport enumeration failed:" << e.what()); + } +#endif } std::shared_ptr<Transport> transport(const std::string & path){ @@ -633,6 +1029,9 @@ namespace trezor{ } } + GenericMessage::GenericMessage(messages::MessageType m_type, const shared_ptr<google::protobuf::Message> &m_msg) + : m_type(m_type), m_msg(m_msg), m_empty(false) {} + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t){ return t.dump(o); } diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp index 7b82fd06f..50c31cf73 100644 --- a/src/device_trezor/trezor/transport.hpp +++ b/src/device_trezor/trezor/transport.hpp @@ -239,6 +239,59 @@ namespace trezor { udp::endpoint m_endpoint; }; +#ifdef WITH_DEVICE_TREZOR_WEBUSB +#include <libusb.h> + + class WebUsbTransport : public Transport { + public: + + explicit WebUsbTransport( + boost::optional<libusb_device_descriptor*> descriptor = boost::none, + boost::optional<std::shared_ptr<Protocol>> proto = boost::none + ); + + virtual ~WebUsbTransport(); + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_device() const; + void require_connected() const; + int get_interface() const; + unsigned char get_endpoint() const; + + int m_conn_count; + std::shared_ptr<Protocol> m_proto; + + libusb_context *m_usb_session; + libusb_device *m_usb_device; + libusb_device_handle *m_usb_device_handle; + std::unique_ptr<libusb_device_descriptor> m_usb_device_desc; + std::vector<uint8_t> m_port_numbers; + int m_bus_id; + int m_device_addr; + +#ifdef WITH_TREZOR_DEBUG + bool m_debug_mode; +#endif + }; + +#endif + // // General helpers // @@ -290,6 +343,20 @@ namespace trezor { [[ noreturn ]] void throw_failure_exception(const messages::common::Failure * failure); /** + * Generic message holder, type + obj + */ + class GenericMessage { + public: + GenericMessage(): m_empty(true) {} + GenericMessage(messages::MessageType m_type, const std::shared_ptr<google::protobuf::Message> &m_msg); + bool empty() const { return m_empty; } + + hw::trezor::messages::MessageType m_type; + std::shared_ptr<google::protobuf::Message> m_msg; + bool m_empty; + }; + + /** * Simple wrapper for write-read message exchange with expected message response type. * * @throws UnexpectedMessageException if the response message type is different than expected. diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp index 951a8f802..30e76eadc 100644 --- a/src/device_trezor/trezor/trezor_defs.hpp +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -27,14 +27,41 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // -#if defined(HAVE_PROTOBUF) && !defined(WITHOUT_TREZOR) - #define WITH_DEVICE_TREZOR 1 -#else - #define WITH_DEVICE_TREZOR 0 +#ifndef USE_DEVICE_TREZOR +#define USE_DEVICE_TREZOR 1 #endif -#ifndef WITH_DEVICE_TREZOR_LITE -#define WITH_DEVICE_TREZOR_LITE 0 +// HAVE_TREZOR_READY set by cmake after protobuf messages +// were generated successfully and all minimal dependencies are met. +#ifndef DEVICE_TREZOR_READY +#undef USE_DEVICE_TREZOR +#define USE_DEVICE_TREZOR 0 +#endif + +#if USE_DEVICE_TREZOR +#define WITH_DEVICE_TREZOR 1 +#endif + +#ifndef WITH_DEVICE_TREZOR +#undef WITH_DEVICE_TREZOR_LITE +#endif + +#if defined(HAVE_TREZOR_LIBUSB) && USE_DEVICE_TREZOR +#define WITH_DEVICE_TREZOR_WEBUSB 1 +#endif + +// Enable / disable UDP in the enumeration +#ifndef USE_DEVICE_TREZOR_UDP +#define USE_DEVICE_TREZOR_UDP 1 +#endif + +// Enable / disable UDP in the enumeration for release +#ifndef USE_DEVICE_TREZOR_UDP_RELEASE +#define USE_DEVICE_TREZOR_UDP_RELEASE 0 +#endif + +#if USE_DEVICE_TREZOR_UDP && (USE_DEVICE_TREZOR_UDP_RELEASE || defined(TREZOR_DEBUG)) +#define WITH_DEVICE_TREZOR_UDP 1 #endif // Avoids protobuf undefined macro warning diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 94071c23c..b1e3bdcd5 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -37,22 +37,14 @@ */ #include <string> -#include <cassert> -#include <map> #include <cstdint> #include <vector> #include <unordered_map> -#include <boost/algorithm/string.hpp> #include "wipeable_string.h" #include "misc_language.h" -#include "crypto/crypto.h" // for declaration of crypto::secret_key -#include <fstream> -#include "common/int-util.h" +#include "int-util.h" #include "mnemonics/electrum-words.h" -#include <stdexcept> -#include <boost/filesystem.hpp> #include <boost/crc.hpp> -#include <boost/algorithm/string/join.hpp> #include "chinese_simplified.h" #include "english.h" diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 5401b9779..60d2c5f15 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -41,7 +41,6 @@ #include <string> #include <cstdint> -#include <map> #include "crypto/crypto.h" // for declaration of crypto::secret_key namespace epee { class wipeable_string; } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 808945393..8930418bd 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -38,7 +38,9 @@ #include "cryptonote_config.h" #include "warnings.h" -#include "net/levin_server_cp2.h" +#include "net/abstract_tcp_server2.h" +#include "net/levin_protocol_handler.h" +#include "net/levin_protocol_handler_async.h" #include "p2p_protocol_defs.h" #include "storages/levin_abstract_invoke2.h" #include "net_peerlist.h" diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index 1592e74e4..d485fb748 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -30,7 +30,9 @@ #include <stdlib.h> #include <boost/thread/mutex.hpp> +#include <boost/thread/lock_guard.hpp> #include "misc_log_ex.h" +#include "span.h" #include "common/perf_timer.h" #include "cryptonote_config.h" extern "C" @@ -218,7 +220,7 @@ static rct::key vector_power_sum(const rct::key &x, size_t n) } /* Given two scalar arrays, construct the inner product */ -static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) +static rct::key inner_product(const epee::span<const rct::key> &a, const epee::span<const rct::key> &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::key res = rct::zero(); @@ -229,6 +231,11 @@ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) return res; } +static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) +{ + return inner_product(epee::span<const rct::key>(a.data(), a.size()), epee::span<const rct::key>(b.data(), b.size())); +} + /* Given two scalar arrays, construct the Hadamard product */ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) { @@ -294,7 +301,7 @@ static rct::keyV vector_subtract(const rct::keyV &a, const rct::key &b) } /* Multiply a scalar and a vector */ -static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) +static rct::keyV vector_scalar(const epee::span<const rct::key> &a, const rct::key &x) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) @@ -304,6 +311,11 @@ static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) return res; } +static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) +{ + return vector_scalar(epee::span<const rct::key>(a.data(), a.size()), x); +} + /* Create a vector from copies of a single value */ static rct::keyV vector_dup(const rct::key &x, size_t N) { @@ -401,17 +413,12 @@ static rct::keyV invert(rct::keyV x) } /* Compute the slice of a vector */ -static rct::keyV slice(const rct::keyV &a, size_t start, size_t stop) +static epee::span<const rct::key> slice(const rct::keyV &a, size_t start, size_t stop) { CHECK_AND_ASSERT_THROW_MES(start < a.size(), "Invalid start index"); CHECK_AND_ASSERT_THROW_MES(stop <= a.size(), "Invalid stop index"); CHECK_AND_ASSERT_THROW_MES(start < stop, "Invalid start/stop indices"); - rct::keyV res(stop - start); - for (size_t i = start; i < stop; ++i) - { - res[i - start] = a[i]; - } - return res; + return epee::span<const rct::key>(&a[start], stop - start); } static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, const rct::key &mash1) diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 8fc42b7e3..d2c4a33cb 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -112,6 +112,7 @@ target_link_libraries(rpc common cryptonote_core cryptonote_protocol + version ${Boost_REGEX_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE @@ -121,6 +122,7 @@ target_link_libraries(daemon_messages LINK_PRIVATE cryptonote_core cryptonote_protocol + version serialization ${EXTRA_LIBRARIES}) @@ -129,6 +131,7 @@ target_link_libraries(daemon_rpc_server rpc cryptonote_core cryptonote_protocol + version daemon_messages serialization ${Boost_CHRONO_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index df9eee781..1bb353bb4 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -91,12 +91,10 @@ namespace cryptonote bool core_rpc_server::init( const boost::program_options::variables_map& vm , const bool restricted - , const network_type nettype , const std::string& port ) { m_restricted = restricted; - m_nettype = nettype; m_net_server.set_threads_prefix("RPC"); auto rpc_config = cryptonote::rpc_args::process(vm); @@ -191,10 +189,13 @@ namespace cryptonote res.rpc_connections_count = get_connections_count(); res.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); res.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); - res.mainnet = m_nettype == MAINNET; - res.testnet = m_nettype == TESTNET; - res.stagenet = m_nettype == STAGENET; - res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; + + cryptonote::network_type net_type = nettype(); + res.mainnet = net_type == MAINNET; + res.testnet = net_type == TESTNET; + res.stagenet = net_type == STAGENET; + res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; + res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); @@ -210,6 +211,7 @@ namespace cryptonote } res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); res.update_available = m_core.is_update_available(); + res.version = m_restricted ? "" : MONERO_VERSION; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -750,7 +752,7 @@ namespace cryptonote PERF_TIMER(on_start_mining); CHECK_CORE_READY(); cryptonote::address_parse_info info; - if(!get_account_address_from_str(info, m_nettype, req.miner_address)) + if(!get_account_address_from_str(info, nettype(), req.miner_address)) { res.status = "Failed, wrong address"; LOG_PRINT_L0(res.status); @@ -831,7 +833,7 @@ namespace cryptonote res.speed = lMiner.get_speed(); res.threads_count = lMiner.get_threads_count(); const account_public_address& lMiningAdr = lMiner.get_mining_address(); - res.address = get_account_address_as_str(m_nettype, false, lMiningAdr); + res.address = get_account_address_as_str(nettype(), false, lMiningAdr); } res.status = CORE_RPC_STATUS_OK; @@ -1064,7 +1066,7 @@ namespace cryptonote cryptonote::address_parse_info info; - if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, m_nettype, req.wallet_address)) + if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, nettype(), req.wallet_address)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_WALLET_ADDRESS; error_resp.message = "Failed to parse wallet address"; @@ -1077,7 +1079,7 @@ namespace cryptonote return false; } - block b = AUTO_VAL_INIT(b); + block b; cryptonote::blobdata blob_reserve; blob_reserve.resize(req.reserve_size, 0); if(!m_core.get_block_template(b, info.address, res.difficulty, res.height, res.expected_reward, blob_reserve)) @@ -1148,7 +1150,7 @@ namespace cryptonote // Fixing of high orphan issue for most pools // Thanks Boolberry! - block b = AUTO_VAL_INIT(b); + block b; if(!parse_and_validate_block_from_blob(blockblob, b)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; @@ -1216,7 +1218,7 @@ namespace cryptonote error_resp.message = "Wrong block blob"; return false; } - block b = AUTO_VAL_INIT(b); + block b; if(!parse_and_validate_block_from_blob(blockblob, b)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; @@ -1590,10 +1592,13 @@ namespace cryptonote res.rpc_connections_count = get_connections_count(); res.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); res.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); - res.mainnet = m_nettype == MAINNET; - res.testnet = m_nettype == TESTNET; - res.stagenet = m_nettype == STAGENET; - res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; + + cryptonote::network_type net_type = nettype(); + res.mainnet = net_type == MAINNET; + res.testnet = net_type == TESTNET; + res.stagenet = net_type == STAGENET; + res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; + res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); @@ -1609,6 +1614,7 @@ namespace cryptonote } res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); res.update_available = m_core.is_update_available(); + res.version = m_restricted ? "" : MONERO_VERSION; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -2133,7 +2139,7 @@ namespace cryptonote return false; } - res.distributions.push_back({std::move(*data), amount, req.binary}); + res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); } } catch (const std::exception &e) @@ -2147,6 +2153,47 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res) + { + PERF_TIMER(on_get_output_distribution_bin); + + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::BIN, "/get_output_distribution.bin", req, res, r)) + return r; + + res.status = "Failed"; + + if (!req.binary) + { + res.status = "Binary only call"; + return false; + } + try + { + // 0 is placeholder for the whole chain + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (uint64_t amount: req.amounts) + { + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + if (!data) + { + res.status = "Failed to get output distribution"; + return false; + } + + res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); + } + } + catch (const std::exception &e) + { + res.status = "Failed to get output distribution"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = { diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 3ba882b23..6a616e2e0 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -70,10 +70,9 @@ namespace cryptonote bool init( const boost::program_options::variables_map& vm, const bool restricted, - const network_type nettype, const std::string& port ); - network_type nettype() const { return m_nettype; } + network_type nettype() const { return m_core.get_nettype(); } CHAIN_HTTP_TO_MAP2(connection_context); //forward http requests to uri map @@ -117,6 +116,7 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH, !m_restricted) MAP_URI_AUTO_JON2("/get_outs", on_get_outs, COMMAND_RPC_GET_OUTPUTS) MAP_URI_AUTO_JON2_IF("/update", on_update, COMMAND_RPC_UPDATE, !m_restricted) + MAP_URI_AUTO_BIN2("/get_output_distribution.bin", on_get_output_distribution_bin, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("get_block_count", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) @@ -187,6 +187,7 @@ namespace cryptonote bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res); + bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res); //json_rpc bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 8e8df7a52..d54bfbf8d 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -34,6 +34,40 @@ #include "cryptonote_basic/difficulty.h" #include "crypto/hash.h" #include "rpc/rpc_handler.h" +#include "common/varint.h" +#include "common/perf_timer.h" + +namespace +{ + template<typename T> + std::string compress_integer_array(const std::vector<T> &v) + { + std::string s; + s.resize(v.size() * (sizeof(T) * 8 / 7 + 1)); + char *ptr = (char*)s.data(); + for (const T &t: v) + tools::write_varint(ptr, t); + s.resize(ptr - s.data()); + return s; + } + + template<typename T> + std::vector<T> decompress_integer_array(const std::string &s) + { + std::vector<T> v; + v.reserve(s.size()); + int read = 0; + const std::string::const_iterator end = s.end(); + for (std::string::const_iterator i = s.begin(); i != end; std::advance(i, read)) + { + T t; + read = tools::read_varint(std::string::const_iterator(i), s.end(), t); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Error decompressing data"); + v.push_back(t); + } + return v; + } +} namespace cryptonote { @@ -50,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 1 +#define CORE_RPC_VERSION_MINOR 2 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -697,9 +731,11 @@ namespace cryptonote struct request { std::vector<get_outputs_out> outputs; + bool get_txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(outputs) + KV_SERIALIZE_OPT(get_txid, true) END_KV_SERIALIZE_MAP() }; @@ -892,6 +928,7 @@ namespace cryptonote bool was_bootstrap_ever_used; uint64_t database_size; bool update_available; + std::string version; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -926,6 +963,7 @@ namespace cryptonote KV_SERIALIZE(was_bootstrap_ever_used) KV_SERIALIZE(database_size) KV_SERIALIZE(update_available) + KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; }; @@ -2222,6 +2260,7 @@ namespace cryptonote uint64_t to_height; bool cumulative; bool binary; + bool compress; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amounts) @@ -2229,6 +2268,7 @@ namespace cryptonote KV_SERIALIZE_OPT(to_height, (uint64_t)0) KV_SERIALIZE_OPT(cumulative, false) KV_SERIALIZE_OPT(binary, true) + KV_SERIALIZE_OPT(compress, false) END_KV_SERIALIZE_MAP() }; @@ -2236,14 +2276,38 @@ namespace cryptonote { rpc::output_distribution_data data; uint64_t amount; + std::string compressed_data; bool binary; + bool compress; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) KV_SERIALIZE_N(data.start_height, "start_height") KV_SERIALIZE(binary) + KV_SERIALIZE(compress) if (this_ref.binary) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + { + if (is_store) + { + if (this_ref.compress) + { + const_cast<std::string&>(this_ref.compressed_data) = compress_integer_array(this_ref.data.distribution); + KV_SERIALIZE(compressed_data) + } + else + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + } + else + { + if (this_ref.compress) + { + KV_SERIALIZE(compressed_data) + const_cast<std::vector<uint64_t>&>(this_ref.data.distribution) = decompress_integer_array<uint64_t>(this_ref.compressed_data); + } + else + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + } + } else KV_SERIALIZE_N(data.distribution, "distribution") KV_SERIALIZE_N(data.base, "base") diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 64a5cc858..e2885dbb5 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -34,6 +34,7 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/blobdatatype.h" #include "ringct/rctSigs.h" +#include "version.h" namespace cryptonote { @@ -437,6 +438,7 @@ namespace rpc res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); + res.info.version = MONERO_VERSION; res.status = Message::STATUS_OK; res.error_details = ""; diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 3b56aff15..e09b6749e 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -191,6 +191,7 @@ namespace rpc uint64_t block_size_median; uint64_t block_weight_median; uint64_t start_time; + std::string version; }; struct output_distribution diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index edd3e6669..a2ff76668 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -27,7 +27,6 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "zmq_server.h" -#include <boost/chrono/chrono.hpp> namespace cryptonote { diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index f906b5d3b..04436c21c 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -113,7 +113,7 @@ struct json_archive; template <> struct json_archive<true> : public json_archive_base<std::ostream, true> { - json_archive(stream_type &s, bool indent = false) : base_type(s, indent) { } + json_archive(stream_type &s, bool indent = false) : base_type(s, indent), inner_array_size_(0) { } template<typename T> static auto promote_to_printable_integer_type(T v) -> decltype(+v) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index d9fd0c13e..6e657b057 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1409,7 +1409,7 @@ bool simple_wallet::print_ring(const std::vector<std::string> &args) crypto::hash txid; if (args.size() != 1) { - fail_msg_writer() << tr("usage: print_ring <key_image|txid>"); + fail_msg_writer() << tr("usage: print_ring <key_image> | <txid>"); return true; } @@ -2348,7 +2348,9 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("incoming_transfers", boost::bind(&simple_wallet::show_incoming_transfers, this, _1), tr("incoming_transfers [available|unavailable] [verbose] [index=<N1>[,<N2>[,...]]]"), - tr("Show the incoming transfers, all or filtered by availability and address index.")); + tr("Show the incoming transfers, all or filtered by availability and address index.\n\n" + "Output format:\n" + "Amount, Spent(\"T\"|\"F\"), \"locked\"|\"unlocked\", RingCT, Global Index, Transaction Hash, Address Index, [Public Key, Key Image] ")); m_cmd_binder.set_handler("payments", boost::bind(&simple_wallet::show_payments, this, _1), tr("payments <PID_1> [<PID_2> ... <PID_N>]"), @@ -2656,7 +2658,9 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("print_ring", boost::bind(&simple_wallet::print_ring, this, _1), tr("print_ring <key_image> | <txid>"), - tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)")); + tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)\n\n" + "Output format:\n" + "Key Image, \"absolute\", list of rings")); m_cmd_binder.set_handler("set_ring", boost::bind(&simple_wallet::set_ring, this, _1), tr("set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"), @@ -4440,7 +4444,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args { if (args.size() > 3) { - fail_msg_writer() << tr("usage: incoming_transfers [available|unavailable] [verbose] [index=<N>]"); + fail_msg_writer() << tr("usage: incoming_transfers [available|unavailable] [verbose] [index=<N1>[,<N2>[,...]]]"); return true; } auto local_args = args; @@ -4482,7 +4486,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args if (local_args.size() > 0) { - fail_msg_writer() << tr("usage: incoming_transfers [available|unavailable] [verbose] [index=<N>]"); + fail_msg_writer() << tr("usage: incoming_transfers [available|unavailable] [verbose] [index=<N1>[,<N2>[,...]]]"); return true; } @@ -6727,11 +6731,7 @@ static std::string get_human_readable_timestamp(uint64_t ts) return "<unknown>"; time_t tt = ts; struct tm tm; -#ifdef WIN32 - gmtime_s(&tm, &tt); -#else - gmtime_r(&tt, &tm); -#endif + epee::misc_utils::get_gmt_time(tt, tm); uint64_t now = time(NULL); uint64_t diff = ts > now ? ts - now : now - ts; strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); @@ -6844,7 +6844,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec std::string note = m_wallet->get_tx_note(pd.m_tx_hash); std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); const std::string type = pd.m_coinbase ? tr("block") : tr("in"); - const bool unlocked = m_wallet->is_tx_spendtime_unlocked(pd.m_unlock_time, pd.m_block_height); + const bool unlocked = m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); transfers.push_back({ pd.m_block_height, pd.m_timestamp, diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 346c052b5..605531e59 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -28,7 +28,6 @@ #include "node_rpc_proxy.h" #include "rpc/core_rpc_server_commands_defs.h" -#include "common/json_util.h" #include "storages/http_abstract_invoke.h" using namespace epee; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index f562d6c06..b69022af4 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -30,6 +30,7 @@ #include <boost/algorithm/string.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/filesystem.hpp> +#include "common/util.h" #include "misc_log_ex.h" #include "misc_language.h" #include "wallet_errors.h" diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 42a5b3fda..202a8ef75 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -67,6 +67,7 @@ using namespace epee; #include "common/json_util.h" #include "memwipe.h" #include "common/base58.h" +#include "common/combinator.h" #include "common/dns_utils.h" #include "common/notify.h" #include "common/perf_timer.h" @@ -177,6 +178,20 @@ namespace return public_keys; } + + bool keys_intersect(const std::unordered_set<crypto::public_key>& s1, const std::unordered_set<crypto::public_key>& s2) + { + if (s1.empty() || s2.empty()) + return false; + + for (const auto& e: s1) + { + if (s2.find(e) != s2.end()) + return true; + } + + return false; + } } namespace @@ -815,7 +830,12 @@ wallet_keys_unlocker::~wallet_keys_unlocker() { if (!locked) return; - w.encrypt_keys(key); + try { w.encrypt_keys(key); } + catch (...) + { + MERROR("Failed to re-encrypt wallet keys"); + // do not propagate through dtor, we'd crash + } } wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): @@ -2149,7 +2169,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry { THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); - tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) @@ -2852,10 +2872,11 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response res = AUTO_VAL_INIT(res); req.amounts.push_back(0); req.from_height = 0; - req.cumulative = true; + req.cumulative = false; req.binary = true; + req.compress = true; m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); if (!r) { @@ -2882,6 +2903,8 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> MWARNING("Failed to request output distribution: results are not for amount 0"); return false; } + for (size_t i = 1; i < res.distributions[0].data.distribution.size(); ++i) + res.distributions[0].data.distribution[i] += res.distributions[0].data.distribution[i-1]; start_height = res.distributions[0].data.start_height; distribution = std::move(res.distributions[0].data.distribution); return true; @@ -3145,7 +3168,6 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account_data = buffer.GetString(); // Encrypt the entire JSON object. - crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string cipher; cipher.resize(account_data.size()); keys_file_data.iv = crypto::rand<crypto::chacha_iv>(); @@ -6043,7 +6065,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto for (auto &sig: ptx.multisig_sigs) { - if (sig.ignore != local_signer) + if (sig.ignore.find(local_signer) == sig.ignore.end()) { ptx.tx.rct_signatures = sig.sigs; @@ -6077,7 +6099,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto bool found = false; for (const auto &sig: ptx.multisig_sigs) { - if (sig.ignore != local_signer && exported_txs.m_signers.find(sig.ignore) == exported_txs.m_signers.end()) + if (sig.ignore.find(local_signer) == sig.ignore.end() && !keys_intersect(sig.ignore, exported_txs.m_signers)) { THROW_WALLET_EXCEPTION_IF(found, error::wallet_internal_error, "More than one transaction is final"); ptx.tx.rct_signatures = sig.sigs; @@ -7186,6 +7208,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } // get the keys for those + req.get_txid = false; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -7517,30 +7540,56 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry // if this is a multisig wallet, create a list of multisig signers we can use std::deque<crypto::public_key> multisig_signers; size_t n_multisig_txes = 0; + std::vector<std::unordered_set<crypto::public_key>> ignore_sets; if (m_multisig && !m_transfers.empty()) { const crypto::public_key local_signer = get_multisig_signer_public_key(); size_t n_available_signers = 1; + + // At this step we need to define set of participants available for signature, + // i.e. those of them who exchanged with multisig info's for (const crypto::public_key &signer: m_multisig_signers) { if (signer == local_signer) continue; - multisig_signers.push_front(signer); for (const auto &i: m_transfers[0].m_multisig_info) { if (i.m_signer == signer) { - multisig_signers.pop_front(); multisig_signers.push_back(signer); ++n_available_signers; break; } } } - multisig_signers.push_back(local_signer); + // n_available_signers includes the transaction creator, but multisig_signers doesn't MDEBUG("We can use " << n_available_signers << "/" << m_multisig_signers.size() << " other signers"); - THROW_WALLET_EXCEPTION_IF(n_available_signers+1 < m_multisig_threshold, error::multisig_import_needed); - n_multisig_txes = n_available_signers == m_multisig_signers.size() ? m_multisig_threshold : 1; + THROW_WALLET_EXCEPTION_IF(n_available_signers < m_multisig_threshold, error::multisig_import_needed); + if (n_available_signers > m_multisig_threshold) + { + // If there more potential signers (those who exchanged with multisig info) + // than threshold needed some of them should be skipped since we don't know + // who will sign tx and who won't. Hence we don't contribute their LR pairs to the signature. + + // We create as many transactions as many combinations of excluded signers may be. + // For example, if we have 2/4 wallet and wallets are: A, B, C and D. Let A be + // transaction creator, so we need just 1 signature from set of B, C, D. + // Using "excluding" logic here we have to exclude 2-of-3 wallets. Combinations go as follows: + // BC, BD, and CD. We save these sets to use later and counting the number of required txs. + tools::Combinator<crypto::public_key> c(std::vector<crypto::public_key>(multisig_signers.begin(), multisig_signers.end())); + auto ignore_combinations = c.combine(multisig_signers.size() + 1 - m_multisig_threshold); + for (const auto& combination: ignore_combinations) + { + ignore_sets.push_back(std::unordered_set<crypto::public_key>(combination.begin(), combination.end())); + } + + n_multisig_txes = ignore_sets.size(); + } + else + { + // If we have exact count of signers just to fit in threshold we don't exclude anyone and create 1 transaction + n_multisig_txes = 1; + } MDEBUG("We will create " << n_multisig_txes << " txes"); } @@ -7608,8 +7657,8 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry src.mask = td.m_mask; if (m_multisig) { - crypto::public_key ignore = m_multisig_threshold == m_multisig_signers.size() ? crypto::null_pkey : multisig_signers.front(); - src.multisig_kLRki = get_multisig_composite_kLRki(idx, ignore, used_L, used_L); + auto ignore_set = ignore_sets.empty() ? std::unordered_set<crypto::public_key>() : ignore_sets.front(); + src.multisig_kLRki = get_multisig_composite_kLRki(idx, ignore_set, used_L, used_L); } else src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()}); @@ -7671,7 +7720,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry std::vector<tools::wallet2::multisig_sig> multisig_sigs; if (m_multisig) { - crypto::public_key ignore = m_multisig_threshold == m_multisig_signers.size() ? crypto::null_pkey : multisig_signers.front(); + auto ignore = ignore_sets.empty() ? std::unordered_set<crypto::public_key>() : ignore_sets.front(); multisig_sigs.push_back({tx.rct_signatures, ignore, used_L, std::unordered_set<crypto::public_key>(), msout}); if (m_multisig_threshold < m_multisig_signers.size()) @@ -7679,7 +7728,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry const crypto::hash prefix_hash = cryptonote::get_transaction_prefix_hash(tx); // create the other versions, one for every other participant (the first one's already done above) - for (size_t signer_index = 1; signer_index < n_multisig_txes; ++signer_index) + for (size_t ignore_index = 1; ignore_index < ignore_sets.size(); ++ignore_index) { std::unordered_set<rct::key> new_used_L; size_t src_idx = 0; @@ -7687,7 +7736,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry for(size_t idx: selected_transfers) { cryptonote::tx_source_entry& src = sources_copy[src_idx]; - src.multisig_kLRki = get_multisig_composite_kLRki(idx, multisig_signers[signer_index], used_L, new_used_L); + src.multisig_kLRki = get_multisig_composite_kLRki(idx, ignore_sets[ignore_index], used_L, new_used_L); ++src_idx; } @@ -7699,7 +7748,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_prefix_hash(ms_tx) != prefix_hash, error::wallet_internal_error, "Multisig txes do not share prefix"); - multisig_sigs.push_back({ms_tx.rct_signatures, multisig_signers[signer_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); + multisig_sigs.push_back({ms_tx.rct_signatures, ignore_sets[ignore_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); ms_tx.rct_signatures = tx.rct_signatures; THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_hash(ms_tx) != cryptonote::get_transaction_hash(tx), error::wallet_internal_error, "Multisig txes differ by more than the signatures"); @@ -11270,7 +11319,7 @@ rct::multisig_kLRki wallet2::get_multisig_kLRki(size_t n, const rct::key &k) con return kLRki; } //---------------------------------------------------------------------------------------------------- -rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const +rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const std::unordered_set<crypto::public_key> &ignore_set, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const { CHECK_AND_ASSERT_THROW_MES(n < m_transfers.size(), "Bad transfer index"); @@ -11281,8 +11330,9 @@ rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const crypto size_t n_signers_used = 1; for (const auto &p: m_transfers[n].m_multisig_info) { - if (p.m_signer == ignore) + if (ignore_set.find(p.m_signer) != ignore_set.end()) continue; + for (const auto &lr: p.m_LR) { if (used_L.find(lr.m_L) != used_L.end()) @@ -11341,7 +11391,10 @@ cryptonote::blobdata wallet2::export_multisig() info[n].m_partial_key_images.push_back(ki); } - size_t nlr = m_multisig_threshold < m_multisig_signers.size() ? m_multisig_threshold - 1 : 1; + // Wallet tries to create as many transactions as many signers combinations. We calculate the maximum number here as follows: + // if we have 2/4 wallet with signers: A, B, C, D and A is a transaction creator it will need to pick up 1 signer from 3 wallets left. + // That means counting combinations for excluding 2-of-3 wallets (k = total signers count - threshold, n = total signers count - 1). + size_t nlr = tools::combinations_count(m_multisig_signers.size() - m_multisig_threshold, m_multisig_signers.size() - 1); for (size_t m = 0; m < nlr; ++m) { td.m_multisig_k.push_back(rct::skGen()); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index eb0763131..0ba8e4c7d 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -37,6 +37,7 @@ #include <boost/serialization/list.hpp> #include <boost/serialization/vector.hpp> #include <boost/serialization/deque.hpp> +#include <boost/thread/lock_guard.hpp> #include <atomic> #include "include_base_utils.h" @@ -49,6 +50,7 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_core/cryptonote_tx_utils.h" #include "common/unordered_containers_boost_serialization.h" +#include "common/util.h" #include "crypto/chacha.h" #include "crypto/hash.h" #include "ringct/rctTypes.h" @@ -374,7 +376,7 @@ namespace tools struct multisig_sig { rct::rctSig sigs; - crypto::public_key ignore; + std::unordered_set<crypto::public_key> ignore; std::unordered_set<rct::key> used_L; std::unordered_set<crypto::public_key> signing_keys; rct::multisig_out msout; @@ -1256,7 +1258,7 @@ namespace tools void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; - rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; + rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const std::unordered_set<crypto::public_key> &ignore_set, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; rct::multisig_kLRki get_multisig_kLRki(size_t n, const rct::key &k) const; rct::key get_multisig_k(size_t idx, const std::unordered_set<rct::key> &used_L) const; void update_multisig_rescan_info(const std::vector<std::vector<rct::key>> &multisig_k, const std::vector<std::vector<tools::wallet2::multisig_info>> &info, size_t n); |