diff options
Diffstat (limited to 'src')
48 files changed, 1958 insertions, 940 deletions
diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 3009b5024..21040a1d8 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -46,6 +46,77 @@ using namespace cryptonote; static bool stop_requested = false; +static bool do_inputs, do_outputs, do_ringsize, do_hours, do_emission, do_fees, do_diff; + +static struct tm prevtm, currtm; +static uint64_t prevsz, currsz; +static uint64_t prevtxs, currtxs; +static uint64_t currblks; +static uint64_t h; +static uint64_t totins, totouts, totrings; +static boost::multiprecision::uint128_t prevemission, prevfees; +static boost::multiprecision::uint128_t emission, fees; +static boost::multiprecision::uint128_t totdiff, mindiff, maxdiff; + +#define MAX_INOUT 0xffffffff +#define MAX_RINGS 0xffffffff + +static uint32_t minins = MAX_INOUT, maxins; +static uint32_t minouts = MAX_INOUT, maxouts; +static uint32_t minrings = MAX_RINGS, maxrings; +static uint32_t io, tottxs; +static uint32_t txhr[24]; + +static void doprint() +{ + char timebuf[64]; + + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); + prevtm = currtm; + std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; + prevsz += currsz; + currsz = 0; + prevtxs += currtxs; + currtxs = 0; + if (!tottxs) + tottxs = 1; + if (do_emission) { + std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission); + prevemission += emission; + emission = 0; + } + if (do_fees) { + std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees); + prevfees += fees; + fees = 0; + } + if (do_diff) { + std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks; + mindiff = 0; maxdiff = 0; totdiff = 0; + } + if (do_inputs) { + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs; + minins = MAX_INOUT; maxins = 0; totins = 0; + } + if (do_outputs) { + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs; + minouts = MAX_INOUT; maxouts = 0; totouts = 0; + } + if (do_ringsize) { + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs; + minrings = MAX_RINGS; maxrings = 0; totrings = 0; + } + if (do_hours) { + for (int i=0; i<24; i++) { + std::cout << "\t" << txhr[i]; + txhr[i] = 0; + } + } + currblks = 0; + tottxs = 0; + std::cout << ENDL; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -123,13 +194,13 @@ int main(int argc, char* argv[]) network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; block_start = command_line::get_arg(vm, arg_block_start); block_stop = command_line::get_arg(vm, arg_block_stop); - bool do_inputs = command_line::get_arg(vm, arg_inputs); - bool do_outputs = command_line::get_arg(vm, arg_outputs); - bool do_ringsize = command_line::get_arg(vm, arg_ringsize); - bool do_hours = command_line::get_arg(vm, arg_hours); - bool do_emission = command_line::get_arg(vm, arg_emission); - bool do_fees = command_line::get_arg(vm, arg_fees); - bool do_diff = command_line::get_arg(vm, arg_diff); + do_inputs = command_line::get_arg(vm, arg_inputs); + do_outputs = command_line::get_arg(vm, arg_outputs); + do_ringsize = command_line::get_arg(vm, arg_ringsize); + do_hours = command_line::get_arg(vm, arg_hours); + do_emission = command_line::get_arg(vm, arg_emission); + do_fees = command_line::get_arg(vm, arg_fees); + do_diff = command_line::get_arg(vm, arg_diff); LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); std::unique_ptr<Blockchain> core_storage; @@ -211,25 +282,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' } std::cout << ENDL; -#define MAX_INOUT 0xffffffff -#define MAX_RINGS 0xffffffff - - struct tm prevtm = {0}, currtm; - uint64_t prevsz = 0, currsz = 0; - uint64_t prevtxs = 0, currtxs = 0; - uint64_t currblks = 0; - uint64_t totins = 0, totouts = 0, totrings = 0; - boost::multiprecision::uint128_t prevemission = 0, prevfees = 0; - boost::multiprecision::uint128_t emission = 0, fees = 0; - boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0; - uint32_t minins = MAX_INOUT, maxins = 0; - uint32_t minouts = MAX_INOUT, maxouts = 0; - uint32_t minrings = MAX_RINGS, maxrings = 0; - uint32_t io, tottxs = 0; - uint32_t txhr[24] = {0}; - unsigned int i; - - for (uint64_t h = block_start; h < block_stop; ++h) + for (h = block_start; h < block_stop; ++h) { cryptonote::blobdata bd = db->get_block_blob_from_height(h); cryptonote::block blk; @@ -239,7 +292,6 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' return 1; } time_t tt = blk.timestamp; - char timebuf[64]; epee::misc_utils::get_gmt_time(tt, currtm); if (!prevtm.tm_year) prevtm = currtm; @@ -247,54 +299,9 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27)) { // check for timestamp fudging around month ends - if (prevtm.tm_mday == 1 && currtm.tm_mday > 27) - goto skip; - strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); - prevtm = currtm; - std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; - prevsz += currsz; - currsz = 0; - prevtxs += currtxs; - currtxs = 0; - if (!tottxs) - tottxs = 1; - if (do_emission) { - std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission); - prevemission += emission; - emission = 0; - } - if (do_fees) { - std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees); - prevfees += fees; - fees = 0; - } - if (do_diff) { - std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks; - mindiff = 0; maxdiff = 0; totdiff = 0; - } - if (do_inputs) { - std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs; - minins = MAX_INOUT; maxins = 0; totins = 0; - } - if (do_outputs) { - std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs; - minouts = MAX_INOUT; maxouts = 0; totouts = 0; - } - if (do_ringsize) { - std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs; - minrings = MAX_RINGS; maxrings = 0; totrings = 0; - } - if (do_hours) { - for (i=0; i<24; i++) { - std::cout << "\t" << txhr[i]; - txhr[i] = 0; - } - } - currblks = 0; - tottxs = 0; - std::cout << ENDL; + if (!(prevtm.tm_mday == 1 && currtm.tm_mday > 27)) + doprint(); } -skip: currsz += bd.size(); uint64_t coinbase_amount; uint64_t tx_fee_amount = 0; @@ -371,6 +378,8 @@ skip: if (stop_requested) break; } + if (currblks) + doprint(); core_storage->deinit(); return 0; diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h deleted file mode 100644 index 4a903107f..000000000 --- a/src/common/boost_serialization_helper.h +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) 2014-2022, The Monero Project -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers - -#pragma once - -#include <boost/archive/binary_iarchive.hpp> -#include <boost/archive/portable_binary_oarchive.hpp> -#include <boost/archive/portable_binary_iarchive.hpp> -#include <boost/filesystem/operations.hpp> - - -namespace tools -{ - template<class t_object> - bool serialize_obj_to_file(t_object& obj, const std::string& file_path) - { - TRY_ENTRY(); -#if defined(_MSC_VER) - // Need to know HANDLE of file to call FlushFileBuffers - HANDLE data_file_handle = ::CreateFile(file_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (INVALID_HANDLE_VALUE == data_file_handle) - return false; - - int data_file_descriptor = _open_osfhandle((intptr_t)data_file_handle, 0); - if (-1 == data_file_descriptor) - { - ::CloseHandle(data_file_handle); - return false; - } - - const std::unique_ptr<FILE, tools::close_file> data_file_file{_fdopen(data_file_descriptor, "wb")}; - if (nullptr == data_file_file) - { - // Call CloseHandle is not necessary - _close(data_file_descriptor); - return false; - } - - // HACK: undocumented constructor, this code may not compile - std::ofstream data_file(data_file_file.get()); - if (data_file.fail()) - { - // Call CloseHandle and _close are not necessary - return false; - } -#else - std::ofstream data_file; - data_file.open(file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); - if (data_file.fail()) - return false; -#endif - - boost::archive::portable_binary_oarchive a(data_file); - a << obj; - if (data_file.fail()) - return false; - - data_file.flush(); -#if defined(_MSC_VER) - // To make sure the file is fully stored on disk - ::FlushFileBuffers(data_file_handle); -#endif - - return true; - CATCH_ENTRY_L0("serialize_obj_to_file", false); - } - - template<class t_object> - bool unserialize_obj_from_file(t_object& obj, const std::string& file_path) - { - TRY_ENTRY(); - - std::ifstream data_file; - data_file.open( file_path, std::ios_base::binary | std::ios_base::in); - if(data_file.fail()) - return false; - try - { - // first try reading in portable mode - boost::archive::portable_binary_iarchive a(data_file); - a >> obj; - } - catch(...) - { - // if failed, try reading in unportable mode - boost::filesystem::copy_file(file_path, file_path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); - data_file.close(); - data_file.open( file_path, std::ios_base::binary | std::ios_base::in); - if(data_file.fail()) - return false; - boost::archive::binary_iarchive a(data_file); - a >> obj; - } - return !data_file.fail(); - CATCH_ENTRY_L0("unserialize_obj_from_file", false); - } -} diff --git a/src/common/data_cache.h b/src/common/data_cache.h new file mode 100644 index 000000000..5e70da115 --- /dev/null +++ b/src/common/data_cache.h @@ -0,0 +1,65 @@ +// Copyright (c) 2014-2022, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include <unordered_set> +#include <mutex> + +namespace tools +{ + template<typename T, size_t MAX_SIZE> + class data_cache + { + public: + void add(const T& value) + { + std::lock_guard<std::mutex> lock(m); + if (data.insert(value).second) + { + T& old_value = buf[counter++ % MAX_SIZE]; + data.erase(old_value); + old_value = value; + } + } + + bool has(const T& value) const + { + std::lock_guard<std::mutex> lock(m); + return (data.find(value) != data.end()); + } + + private: + mutable std::mutex m; + std::unordered_set<T> data; + T buf[MAX_SIZE] = {}; + size_t counter = 0; + }; +} diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index e00421f87..466224390 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -30,6 +30,8 @@ // check local first (in the event of static or in-source compilation of libunbound) #include "unbound.h" +#include <deque> +#include <set> #include <stdlib.h> #include "include_base_utils.h" #include "common/threadpool.h" @@ -326,11 +328,6 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec dnssec_available = false; dnssec_valid = false; - if (!check_address_syntax(url.c_str())) - { - return addresses; - } - // destructor takes care of cleanup ub_result_ptr result; @@ -413,16 +410,6 @@ DNSResolver DNSResolver::create() return DNSResolver(); } -bool DNSResolver::check_address_syntax(const char *addr) const -{ - // if string doesn't contain a dot, we won't consider it a url for now. - if (strchr(addr,'.') == NULL) - { - return false; - } - return true; -} - namespace dns_utils { diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index f9507b42a..81079ba30 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -159,15 +159,6 @@ private: // TODO: modify this to accommodate DNSSEC std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); - /** - * @brief Checks a string to see if it looks like a URL - * - * @param addr the string to be checked - * - * @return true if it looks enough like a URL, false if not - */ - bool check_address_syntax(const char *addr) const; - DNSResolverData *m_data; }; // class DNSResolver diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 53421e18b..fcf8ca945 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -31,6 +31,7 @@ #include <boost/thread/mutex.hpp> #include <boost/thread/thread.hpp> #include <cstddef> +#include <deque> #include <functional> #include <utility> #include <vector> diff --git a/src/crypto/c_threads.h b/src/crypto/c_threads.h index c5431cb8d..3457738b3 100644 --- a/src/crypto/c_threads.h +++ b/src/crypto/c_threads.h @@ -30,29 +30,41 @@ #pragma once #ifdef _WIN32 + #include <windows.h> -#define CTHR_MUTEX_TYPE HANDLE -#define CTHR_MUTEX_INIT NULL -#define CTHR_MUTEX_LOCK(x) do { if (x == NULL) { \ - HANDLE p = CreateMutex(NULL, FALSE, NULL); \ - if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \ - CloseHandle(p); \ - } WaitForSingleObject(x, INFINITE); } while(0) -#define CTHR_MUTEX_UNLOCK(x) ReleaseMutex(x) + +#define CTHR_RWLOCK_TYPE SRWLOCK +#define CTHR_RWLOCK_INIT SRWLOCK_INIT +#define CTHR_RWLOCK_LOCK_WRITE(x) AcquireSRWLockExclusive(&x) +#define CTHR_RWLOCK_UNLOCK_WRITE(x) ReleaseSRWLockExclusive(&x) +#define CTHR_RWLOCK_LOCK_READ(x) AcquireSRWLockShared(&x) +#define CTHR_RWLOCK_UNLOCK_READ(x) ReleaseSRWLockShared(&x) +#define CTHR_RWLOCK_TRYLOCK_READ(x) TryAcquireSRWLockShared(&x) + #define CTHR_THREAD_TYPE HANDLE -#define CTHR_THREAD_RTYPE void -#define CTHR_THREAD_RETURN return -#define CTHR_THREAD_CREATE(thr, func, arg) thr = (HANDLE)_beginthread(func, 0, arg) -#define CTHR_THREAD_JOIN(thr) WaitForSingleObject(thr, INFINITE) +#define CTHR_THREAD_RTYPE unsigned __stdcall +#define CTHR_THREAD_RETURN _endthreadex(0); return 0; +#define CTHR_THREAD_CREATE(thr, func, arg) ((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L) +#define CTHR_THREAD_JOIN(thr) do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0) +#define CTHR_THREAD_CLOSE(thr) CloseHandle((HANDLE)thr); + #else + #include <pthread.h> -#define CTHR_MUTEX_TYPE pthread_mutex_t -#define CTHR_MUTEX_INIT PTHREAD_MUTEX_INITIALIZER -#define CTHR_MUTEX_LOCK(x) pthread_mutex_lock(&x) -#define CTHR_MUTEX_UNLOCK(x) pthread_mutex_unlock(&x) + +#define CTHR_RWLOCK_TYPE pthread_rwlock_t +#define CTHR_RWLOCK_INIT PTHREAD_RWLOCK_INITIALIZER +#define CTHR_RWLOCK_LOCK_WRITE(x) pthread_rwlock_wrlock(&x) +#define CTHR_RWLOCK_UNLOCK_WRITE(x) pthread_rwlock_unlock(&x) +#define CTHR_RWLOCK_LOCK_READ(x) pthread_rwlock_rdlock(&x) +#define CTHR_RWLOCK_UNLOCK_READ(x) pthread_rwlock_unlock(&x) +#define CTHR_RWLOCK_TRYLOCK_READ(x) (pthread_rwlock_tryrdlock(&x) == 0) + #define CTHR_THREAD_TYPE pthread_t #define CTHR_THREAD_RTYPE void * #define CTHR_THREAD_RETURN return NULL -#define CTHR_THREAD_CREATE(thr, func, arg) pthread_create(&thr, NULL, func, arg) +#define CTHR_THREAD_CREATE(thr, func, arg) (pthread_create(&thr, NULL, func, arg) == 0) #define CTHR_THREAD_JOIN(thr) pthread_join(thr, NULL) +#define CTHR_THREAD_CLOSE(thr) + #endif diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 4b392d472..971bf663f 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -38,7 +38,6 @@ DISABLE_VS_WARNINGS(4146 4244) /* Predeclarations */ -static void fe_mul(fe, const fe, const fe); static void fe_sq(fe, const fe); static void ge_madd(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_msub(ge_p1p1 *, const ge_p3 *, const ge_precomp *); @@ -72,7 +71,7 @@ uint64_t load_4(const unsigned char *in) h = 0 */ -static void fe_0(fe h) { +void fe_0(fe h) { h[0] = 0; h[1] = 0; h[2] = 0; @@ -375,7 +374,7 @@ Can get away with 11 carries, but then data flow is much deeper. With tighter constraints on inputs can squeeze carries into int32. */ -static void fe_mul(fe h, const fe f, const fe g) { +void fe_mul(fe h, const fe f, const fe g) { int32_t f0 = f[0]; int32_t f1 = f[1]; int32_t f2 = f[2]; diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index e4901e080..7a6f0789a 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -30,6 +30,8 @@ #pragma once +#include <stdint.h> + /* From fe.h */ typedef int32_t fe[10]; @@ -161,5 +163,7 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); +void fe_mul(fe out, const fe, const fe); +void fe_0(fe h); int ge_p3_is_point_at_infinity_vartime(const ge_p3 *p); diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index d8cd6c6a0..43ea59ac6 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -335,8 +335,16 @@ namespace crypto { inline bool operator<(const public_key &p1, const public_key &p2) { return memcmp(&p1, &p2, sizeof(public_key)) < 0; } inline bool operator>(const public_key &p1, const public_key &p2) { return p2 < p1; } + inline bool operator<(const key_image &p1, const key_image &p2) { return memcmp(&p1, &p2, sizeof(key_image)) < 0; } + inline bool operator>(const key_image &p1, const key_image &p2) { return p2 < p1; } } +// type conversions for easier calls to sc_add(), sc_sub(), hash functions +inline unsigned char* to_bytes(crypto::ec_scalar &scalar) { return &reinterpret_cast<unsigned char&>(scalar); } +inline const unsigned char* to_bytes(const crypto::ec_scalar &scalar) { return &reinterpret_cast<const unsigned char&>(scalar); } +inline unsigned char* to_bytes(crypto::ec_point &point) { return &reinterpret_cast<unsigned char&>(point); } +inline const unsigned char* to_bytes(const crypto::ec_point &point) { return &reinterpret_cast<const unsigned char&>(point); } + CRYPTO_MAKE_HASHABLE(public_key) CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(public_key_memsafe) diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index b7ec80d7c..9d3abc3f8 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -97,5 +97,9 @@ void rx_slow_hash_allocate_state(void); void rx_slow_hash_free_state(void); uint64_t rx_seedheight(const uint64_t height); void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height); -void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash, int miners, int is_alt); -void rx_reorg(const uint64_t split_height); + +void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads); +void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash); + +void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads); +uint32_t rx_get_miner_thread(void); diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index 40ef96ac9..672144fcf 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -43,32 +43,41 @@ #define RX_LOGCAT "randomx" +// Report large page allocation failures as debug messages +#define alloc_err_msg(x) mdebug(RX_LOGCAT, x); + +static CTHR_RWLOCK_TYPE main_dataset_lock = CTHR_RWLOCK_INIT; +static CTHR_RWLOCK_TYPE main_cache_lock = CTHR_RWLOCK_INIT; + +static randomx_dataset *main_dataset = NULL; +static randomx_cache *main_cache = NULL; +static char main_seedhash[HASH_SIZE]; +static int main_seedhash_set = 0; + +static CTHR_RWLOCK_TYPE secondary_cache_lock = CTHR_RWLOCK_INIT; + +static randomx_cache *secondary_cache = NULL; +static char secondary_seedhash[HASH_SIZE]; +static int secondary_seedhash_set = 0; + #if defined(_MSC_VER) #define THREADV __declspec(thread) #else #define THREADV __thread #endif -typedef struct rx_state { - CTHR_MUTEX_TYPE rs_mutex; - char rs_hash[HASH_SIZE]; - uint64_t rs_height; - randomx_cache *rs_cache; -} rx_state; - -static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT; -static CTHR_MUTEX_TYPE rx_dataset_mutex = CTHR_MUTEX_INIT; +static THREADV randomx_vm *main_vm_full = NULL; +static THREADV randomx_vm *main_vm_light = NULL; +static THREADV randomx_vm *secondary_vm_light = NULL; -static rx_state rx_s[2] = {{CTHR_MUTEX_INIT,{0},0,0},{CTHR_MUTEX_INIT,{0},0,0}}; +static THREADV uint32_t miner_thread = 0; -static randomx_dataset *rx_dataset; -static int rx_dataset_nomem; -static int rx_dataset_nolp; -static uint64_t rx_dataset_height; -static THREADV randomx_vm *rx_vm = NULL; +static bool is_main(const char* seedhash) { return main_seedhash_set && (memcmp(seedhash, main_seedhash, HASH_SIZE) == 0); } +static bool is_secondary(const char* seedhash) { return secondary_seedhash_set && (memcmp(seedhash, secondary_seedhash, HASH_SIZE) == 0); } static void local_abort(const char *msg) { + merror(RX_LOGCAT, "%s", msg); fprintf(stderr, "%s\n", msg); #ifdef NDEBUG _exit(1); @@ -77,6 +86,16 @@ static void local_abort(const char *msg) #endif } +static void hash2hex(const char* hash, char* hex) { + const char* d = "0123456789abcdef"; + for (int i = 0; i < HASH_SIZE; ++i) { + const uint8_t b = hash[i]; + hex[i * 2 + 0] = d[b >> 4]; + hex[i * 2 + 1] = d[b & 15]; + } + hex[HASH_SIZE * 2] = '\0'; +} + static inline int disabled_flags(void) { static int flags = -1; @@ -157,19 +176,6 @@ static unsigned int get_seedhash_epoch_blocks(void) return blocks; } -void rx_reorg(const uint64_t split_height) { - int i; - CTHR_MUTEX_LOCK(rx_mutex); - for (i=0; i<2; i++) { - if (split_height <= rx_s[i].rs_height) { - if (rx_s[i].rs_height == rx_dataset_height) - rx_dataset_height = 1; - rx_s[i].rs_height = 1; /* set to an invalid seed height */ - } - } - CTHR_MUTEX_UNLOCK(rx_mutex); -} - uint64_t rx_seedheight(const uint64_t height) { const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag(); const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks(); @@ -183,6 +189,103 @@ void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nexth *nextheight = rx_seedheight(height + get_seedhash_epoch_lag()); } +static void rx_alloc_dataset(randomx_flags flags, randomx_dataset** dataset, int ignore_env) +{ + if (*dataset) { + return; + } + + if (disabled_flags() & RANDOMX_FLAG_FULL_MEM) { + static int shown = 0; + if (!shown) { + shown = 1; + minfo(RX_LOGCAT, "RandomX dataset is disabled by MONERO_RANDOMX_UMASK environment variable."); + } + return; + } + + if (!ignore_env && !getenv("MONERO_RANDOMX_FULL_MEM")) { + static int shown = 0; + if (!shown) { + shown = 1; + minfo(RX_LOGCAT, "RandomX dataset is not enabled by default. Use MONERO_RANDOMX_FULL_MEM environment variable to enable it."); + } + return; + } + + *dataset = randomx_alloc_dataset((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags()); + if (!*dataset) { + alloc_err_msg("Couldn't allocate RandomX dataset using large pages"); + *dataset = randomx_alloc_dataset(flags & ~disabled_flags()); + if (!*dataset) { + merror(RX_LOGCAT, "Couldn't allocate RandomX dataset"); + } + } +} + +static void rx_alloc_cache(randomx_flags flags, randomx_cache** cache) +{ + if (*cache) { + return; + } + + *cache = randomx_alloc_cache((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags()); + if (!*cache) { + alloc_err_msg("Couldn't allocate RandomX cache using large pages"); + *cache = randomx_alloc_cache(flags & ~disabled_flags()); + if (!*cache) local_abort("Couldn't allocate RandomX cache"); + } +} + +static void rx_init_full_vm(randomx_flags flags, randomx_vm** vm) +{ + if (*vm || !main_dataset || (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) { + return; + } + + if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) { + flags |= RANDOMX_FLAG_SECURE; + } + + *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset); + if (!*vm) { + static int shown = 0; + if (!shown) { + shown = 1; + alloc_err_msg("Couldn't allocate RandomX full VM using large pages (will print only once)"); + } + *vm = randomx_create_vm((flags | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset); + if (!*vm) { + merror(RX_LOGCAT, "Couldn't allocate RandomX full VM"); + } + } +} + +static void rx_init_light_vm(randomx_flags flags, randomx_vm** vm, randomx_cache* cache) +{ + if (*vm) { + randomx_vm_set_cache(*vm, cache); + return; + } + + if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) { + flags |= RANDOMX_FLAG_SECURE; + } + + flags &= ~RANDOMX_FLAG_FULL_MEM; + + *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags(), cache, NULL); + if (!*vm) { + static int shown = 0; + if (!shown) { + shown = 1; + alloc_err_msg("Couldn't allocate RandomX light VM using large pages (will print only once)"); + } + *vm = randomx_create_vm(flags & ~disabled_flags(), cache, NULL); + if (!*vm) local_abort("Couldn't allocate RandomX light VM"); + } +} + typedef struct seedinfo { randomx_cache *si_cache; unsigned long si_start; @@ -191,187 +294,231 @@ typedef struct seedinfo { static CTHR_THREAD_RTYPE rx_seedthread(void *arg) { seedinfo *si = arg; - randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count); + randomx_init_dataset(main_dataset, si->si_cache, si->si_start, si->si_count); CTHR_THREAD_RETURN; } -static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_t seedheight) { - if (miners > 1) { - unsigned long delta = randomx_dataset_item_count() / miners; - unsigned long start = 0; - int i; - seedinfo *si; - CTHR_THREAD_TYPE *st; - si = malloc(miners * sizeof(seedinfo)); - if (si == NULL) - local_abort("Couldn't allocate RandomX mining threadinfo"); - st = malloc(miners * sizeof(CTHR_THREAD_TYPE)); - if (st == NULL) { - free(si); - local_abort("Couldn't allocate RandomX mining threadlist"); - } - for (i=0; i<miners-1; i++) { - si[i].si_cache = rs_cache; - si[i].si_start = start; - si[i].si_count = delta; - start += delta; - } - si[i].si_cache = rs_cache; +static void rx_init_dataset(size_t max_threads) { + if (!main_dataset) { + return; + } + + // leave 2 CPU cores for other tasks + const size_t num_threads = (max_threads < 4) ? 1 : (max_threads - 2); + seedinfo* si = malloc(num_threads * sizeof(seedinfo)); + if (!si) local_abort("Couldn't allocate RandomX mining threadinfo"); + + const uint32_t delta = randomx_dataset_item_count() / num_threads; + uint32_t start = 0; + + const size_t n1 = num_threads - 1; + for (size_t i = 0; i < n1; ++i) { + si[i].si_cache = main_cache; si[i].si_start = start; - si[i].si_count = randomx_dataset_item_count() - start; - for (i=1; i<miners; i++) { - CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i]); - } - randomx_init_dataset(rx_dataset, rs_cache, 0, si[0].si_count); - for (i=1; i<miners; i++) { - CTHR_THREAD_JOIN(st[i]); + si[i].si_count = delta; + start += delta; + } + + si[n1].si_cache = main_cache; + si[n1].si_start = start; + si[n1].si_count = randomx_dataset_item_count() - start; + + CTHR_THREAD_TYPE *st = malloc(num_threads * sizeof(CTHR_THREAD_TYPE)); + if (!st) local_abort("Couldn't allocate RandomX mining threadlist"); + + CTHR_RWLOCK_LOCK_READ(main_cache_lock); + for (size_t i = 0; i < n1; ++i) { + if (!CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i])) { + local_abort("Couldn't start RandomX seed thread"); } - free(st); - free(si); - } else { - randomx_init_dataset(rx_dataset, rs_cache, 0, randomx_dataset_item_count()); } - rx_dataset_height = seedheight; + randomx_init_dataset(main_dataset, si[n1].si_cache, si[n1].si_start, si[n1].si_count); + for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]); + CTHR_RWLOCK_UNLOCK_READ(main_cache_lock); + + free(st); + free(si); + + minfo(RX_LOGCAT, "RandomX dataset initialized"); } -void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, - char *hash, int miners, int is_alt) { - uint64_t s_height = rx_seedheight(mainheight); - int toggle = (s_height & get_seedhash_epoch_blocks()) != 0; - randomx_flags flags = enabled_flags() & ~disabled_flags(); - rx_state *rx_sp; - randomx_cache *cache; - - CTHR_MUTEX_LOCK(rx_mutex); - - /* if alt block but with same seed as mainchain, no need for alt cache */ - if (is_alt) { - if (s_height == seedheight && !memcmp(rx_s[toggle].rs_hash, seedhash, HASH_SIZE)) - is_alt = 0; - } else { - /* RPC could request an earlier block on mainchain */ - if (s_height > seedheight) - is_alt = 1; - /* miner can be ahead of mainchain */ - else if (s_height < seedheight) - toggle ^= 1; - } - - toggle ^= (is_alt != 0); - - rx_sp = &rx_s[toggle]; - CTHR_MUTEX_LOCK(rx_sp->rs_mutex); - CTHR_MUTEX_UNLOCK(rx_mutex); - - cache = rx_sp->rs_cache; - if (cache == NULL) { - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { - cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); - if (cache == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache"); - } - } - if (cache == NULL) { - cache = randomx_alloc_cache(flags); - if (cache == NULL) - local_abort("Couldn't allocate RandomX cache"); - } +typedef struct thread_info { + char seedhash[HASH_SIZE]; + size_t max_threads; +} thread_info; + +static CTHR_THREAD_RTYPE rx_set_main_seedhash_thread(void *arg) { + thread_info* info = arg; + + CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock); + CTHR_RWLOCK_LOCK_WRITE(main_cache_lock); + + // Double check that seedhash wasn't already updated + if (is_main(info->seedhash)) { + CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock); + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + free(info); + CTHR_THREAD_RETURN; } - if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) { - randomx_init_cache(cache, seedhash, HASH_SIZE); - rx_sp->rs_cache = cache; - rx_sp->rs_height = seedheight; - memcpy(rx_sp->rs_hash, seedhash, HASH_SIZE); + memcpy(main_seedhash, info->seedhash, HASH_SIZE); + main_seedhash_set = 1; + + char buf[HASH_SIZE * 2 + 1]; + hash2hex(main_seedhash, buf); + minfo(RX_LOGCAT, "RandomX new main seed hash is %s", buf); + + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + rx_alloc_dataset(flags, &main_dataset, 0); + rx_alloc_cache(flags, &main_cache); + + randomx_init_cache(main_cache, info->seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX main cache initialized"); + + CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock); + + // From this point, rx_slow_hash can calculate hashes in light mode, but dataset is not initialized yet + rx_init_dataset(info->max_threads); + + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + + free(info); + CTHR_THREAD_RETURN; +} + +void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads) { + // Early out if seedhash didn't change + if (is_main(seedhash)) { + return; } - if (rx_vm == NULL) { - if ((flags & RANDOMX_FLAG_JIT) && !miners) { - flags |= RANDOMX_FLAG_SECURE & ~disabled_flags(); - } - if (miners && (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) { - miners = 0; - } - if (miners) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (!rx_dataset_nomem) { - if (rx_dataset == NULL) { - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); - if (rx_dataset == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); - } - } - if (rx_dataset == NULL) - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); - if (rx_dataset != NULL) - rx_initdata(rx_sp->rs_cache, miners, seedheight); - } - } - if (rx_dataset != NULL) - flags |= RANDOMX_FLAG_FULL_MEM; - else { - miners = 0; - if (!rx_dataset_nomem) { - rx_dataset_nomem = 1; - mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner"); + + // Update main cache and dataset in the background + thread_info* info = malloc(sizeof(thread_info)); + if (!info) local_abort("Couldn't allocate RandomX mining threadinfo"); + + memcpy(info->seedhash, seedhash, HASH_SIZE); + info->max_threads = max_dataset_init_threads; + + CTHR_THREAD_TYPE t; + if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) { + local_abort("Couldn't start RandomX seed thread"); + } + CTHR_THREAD_CLOSE(t); +} + +void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) { + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + int success = 0; + + // Fast path (seedhash == main_seedhash) + // Multiple threads can run in parallel in fast or light mode, 1-2 ms or 10-15 ms per hash per thread + if (is_main(seedhash)) { + // If CTHR_RWLOCK_TRYLOCK_READ fails it means dataset is being initialized now, so use the light mode + if (main_dataset && CTHR_RWLOCK_TRYLOCK_READ(main_dataset_lock)) { + // Double check that main_seedhash didn't change + if (is_main(seedhash)) { + rx_init_full_vm(flags, &main_vm_full); + if (main_vm_full) { + randomx_calculate_hash(main_vm_full, data, length, result_hash); + success = 1; } } - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); - } - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES) && !rx_dataset_nolp) { - rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) { //large pages failed - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); - rx_dataset_nolp = 1; + CTHR_RWLOCK_UNLOCK_READ(main_dataset_lock); + } else { + CTHR_RWLOCK_LOCK_READ(main_cache_lock); + // Double check that main_seedhash didn't change + if (is_main(seedhash)) { + rx_init_light_vm(flags, &main_vm_light, main_cache); + randomx_calculate_hash(main_vm_light, data, length, result_hash); + success = 1; } + CTHR_RWLOCK_UNLOCK_READ(main_cache_lock); } - if (rx_vm == NULL) - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) {//fallback if everything fails - flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); - } - if (rx_vm == NULL) - local_abort("Couldn't allocate RandomX VM"); - } else if (miners) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (rx_dataset != NULL && rx_dataset_height != seedheight) - rx_initdata(cache, miners, seedheight); - else if (rx_dataset == NULL) { - /* this is a no-op if the cache hasn't changed */ - randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); + } + + if (success) { + return; + } + + char buf[HASH_SIZE * 2 + 1]; + + // Slow path (seedhash != main_seedhash, but seedhash == secondary_seedhash) + // Multiple threads can run in parallel in light mode, 10-15 ms per hash per thread + if (!secondary_cache) { + CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock); + if (!secondary_cache) { + hash2hex(seedhash, buf); + minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf); + + rx_alloc_cache(flags, &secondary_cache); + randomx_init_cache(secondary_cache, seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX secondary cache updated"); + memcpy(secondary_seedhash, seedhash, HASH_SIZE); + secondary_seedhash_set = 1; } - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); - } else { - /* this is a no-op if the cache hasn't changed */ - randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); - } - /* mainchain users can run in parallel */ - if (!is_alt) - CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex); - randomx_calculate_hash(rx_vm, data, length, hash); - /* altchain slot users always get fully serialized */ - if (is_alt) - CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex); -} + CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock); + } + + CTHR_RWLOCK_LOCK_READ(secondary_cache_lock); + if (is_secondary(seedhash)) { + rx_init_light_vm(flags, &secondary_vm_light, secondary_cache); + randomx_calculate_hash(secondary_vm_light, data, length, result_hash); + success = 1; + } + CTHR_RWLOCK_UNLOCK_READ(secondary_cache_lock); -void rx_slow_hash_allocate_state(void) { + if (success) { + return; + } + + // Slowest path (seedhash != main_seedhash, seedhash != secondary_seedhash) + // Only one thread runs at a time and updates secondary_seedhash if needed, up to 200-500 ms per hash + CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock); + if (!is_secondary(seedhash)) { + hash2hex(seedhash, buf); + minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf); + + randomx_init_cache(secondary_cache, seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX secondary cache updated"); + memcpy(secondary_seedhash, seedhash, HASH_SIZE); + secondary_seedhash_set = 1; + } + rx_init_light_vm(flags, &secondary_vm_light, secondary_cache); + randomx_calculate_hash(secondary_vm_light, data, length, result_hash); + CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock); } -void rx_slow_hash_free_state(void) { - if (rx_vm != NULL) { - randomx_destroy_vm(rx_vm); - rx_vm = NULL; +void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads) { + miner_thread = value; + + // If dataset is not allocated yet, try to allocate and initialize it + CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock); + if (main_dataset) { + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + return; } + + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + rx_alloc_dataset(flags, &main_dataset, 1); + rx_init_dataset(max_dataset_init_threads); + + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); +} + +uint32_t rx_get_miner_thread() { + return miner_thread; } -void rx_stop_mining(void) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (rx_dataset != NULL) { - randomx_dataset *rd = rx_dataset; - rx_dataset = NULL; - randomx_release_dataset(rd); +void rx_slow_hash_allocate_state() {} + +static void rx_destroy_vm(randomx_vm** vm) { + if (*vm) { + randomx_destroy_vm(*vm); + *vm = NULL; } - rx_dataset_nomem = 0; - rx_dataset_nolp = 0; - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); +} + +void rx_slow_hash_free_state() { + rx_destroy_vm(&main_vm_full); + rx_destroy_vm(&main_vm_light); + rx_destroy_vm(&secondary_vm_light); } diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 5b0db9518..98f1555b6 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -82,6 +82,7 @@ using namespace epee; #include "miner.h" +#include "crypto/hash.h" extern "C" void slow_hash_allocate_state(); @@ -436,7 +437,6 @@ namespace cryptonote { m_stop = true; } - extern "C" void rx_stop_mining(void); //----------------------------------------------------------------------------------------------------- bool miner::stop() { @@ -469,7 +469,6 @@ namespace cryptonote MINFO("Mining has been stopped, " << m_threads.size() << " finished" ); m_threads.clear(); m_threads_autodetect.clear(); - rx_stop_mining(); return true; } //----------------------------------------------------------------------------------------------------- @@ -524,6 +523,8 @@ namespace cryptonote bool miner::worker_thread() { const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment + crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency()); + MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]"); MGINFO("Miner thread was started ["<< th_local_index << "]"); uint32_t nonce = m_starter_nonce + th_local_index; diff --git a/src/cryptonote_basic/verification_context.h b/src/cryptonote_basic/verification_context.h index 34157218f..10a16a8c1 100644 --- a/src/cryptonote_basic/verification_context.h +++ b/src/cryptonote_basic/verification_context.h @@ -42,7 +42,12 @@ namespace cryptonote static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none"); relay_method m_relay; // gives indication on how tx should be relayed (if at all) - bool m_verifivation_failed; //bad tx, should drop connection + bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense + // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop + // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be + // confused with breaking soft fork rules, because tx could be later added to the chain if mined + // because it does not violate consensus rules. + bool m_no_drop_offense; bool m_verifivation_impossible; //the transaction is related with an alternative blockchain bool m_added_to_pool; bool m_low_mixin; @@ -53,6 +58,7 @@ namespace cryptonote bool m_overspend; bool m_fee_too_low; bool m_too_few_outputs; + bool m_tx_extra_too_big; }; struct block_verification_context diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 2ec194ef8..bac49aa94 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -206,6 +206,11 @@ #define DNS_BLOCKLIST_LIFETIME (86400 * 8) +//The limit is enough for the mandatory transaction content with 16 outputs (547 bytes), +//a custom tag (1 byte) and up to 32 bytes of custom data for each recipient. +// (1+32) + (1+1+16*32) + (1+16*32) = 1060 +#define MAX_TX_EXTRA_SIZE 1060 + // New constants are intended to go here namespace config { @@ -248,6 +253,7 @@ namespace config const unsigned char HASH_KEY_MM_SLOT = 'm'; const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS_SEED[] = "multisig_tx_privkeys_seed"; const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS[] = "multisig_tx_privkeys"; + const constexpr char HASH_KEY_TXHASH_AND_MIXRING[] = "txhash_and_mixring"; // Multisig const uint32_t MULTISIG_MAX_SIGNERS{16}; diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 69411e379..beead6217 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -31,7 +31,9 @@ set(cryptonote_core_sources cryptonote_core.cpp tx_pool.cpp tx_sanity_check.cpp - cryptonote_tx_utils.cpp) + cryptonote_tx_utils.cpp + tx_verification_utils.cpp +) set(cryptonote_core_headers) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 135dd3df0..ba1f22d91 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -48,7 +48,6 @@ #include "file_io_utils.h" #include "int-util.h" #include "common/threadpool.h" -#include "common/boost_serialization_helper.h" #include "warnings.h" #include "crypto/hash.h" #include "cryptonote_core.h" @@ -57,6 +56,7 @@ #include "common/notify.h" #include "common/varint.h" #include "common/pruning.h" +#include "common/data_cache.h" #include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -98,7 +98,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_difficulty_for_next_block(1), m_btc_valid(false), m_batch_success(true), - m_prepare_height(0) + m_prepare_height(0), + m_rct_ver_cache() { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -456,6 +457,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline if (!update_next_cumulative_weight_limit()) return false; } + + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height())); + if (seedhash != crypto::null_hash) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + } + return true; } //------------------------------------------------------------------ @@ -570,6 +579,12 @@ void Blockchain::pop_blocks(uint64_t nblocks) if (stop_batch) m_db->batch_stop(); + + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height())); + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + } } //------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the @@ -1239,18 +1254,20 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> } m_hardfork->reorganize_from_chain_height(split_height); - get_block_longhash_reorg(split_height); std::shared_ptr<tools::Notify> reorg_notify = m_reorg_notify; if (reorg_notify) reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + const uint64_t new_height = m_db->height(); + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height)); + crypto::hash prev_id; if (!get_block_hash(alt_chain.back().bl, prev_id)) MERROR("Failed to get block hash of an alternative chain's tip"); else - send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + send_miner_notifications(new_height, seedhash, prev_id, alt_chain.back().already_generated_coins); for (const auto& notifier : m_block_notifiers) { @@ -1262,6 +1279,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> } } + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height()); return true; } @@ -2001,7 +2021,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { seedhash = get_block_id_by_height(seedheight); } - get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash); + get_altblock_longhash(bei.bl, proof_of_work, seedhash); } else { get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0); @@ -2044,7 +2064,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id cryptonote::blobdata blob; if (m_tx_pool.have_tx(txid, relay_category::legacy)) { - if (m_tx_pool.get_transaction_info(txid, td)) + if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/)) { bei.block_cumulative_weight += td.weight; } @@ -3192,7 +3212,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const } return false; } -bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const +bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) { PERF_TIMER(expand_transaction_2); CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2"); @@ -3515,6 +3535,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, false, "Transaction spends at least one output which is too young"); } + // Warn that new RCT types are present, and thus the cache is not being used effectively + static constexpr const std::uint8_t RCT_CACHE_TYPE = rct::RCTTypeBulletproofPlus; + if (tx.rct_signatures.type > RCT_CACHE_TYPE) + { + MWARNING("RCT cache is not caching new verification results. Please update RCT_CACHE_TYPE!"); + } + if (tx.version == 1) { if (threads > 1) @@ -3536,12 +3563,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } else { - if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys)) - { - MERROR_VER("Failed to expand rct signatures!"); - return false; - } - // from version 2, check ringct signatures // obviously, the original and simple rct APIs use a mixRing that's indexes // in opposite orders, because it'd be too simple otherwise... @@ -3559,61 +3580,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, case rct::RCTTypeCLSAG: case rct::RCTTypeBulletproofPlus: { - // check all this, either reconstructed (so should really pass), or not - { - if (pubkeys.size() != rv.mixRing.size()) - { - MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size"); - return false; - } - for (size_t i = 0; i < pubkeys.size(); ++i) - { - if (pubkeys[i].size() != rv.mixRing[i].size()) - { - MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size"); - return false; - } - } - - for (size_t n = 0; n < pubkeys.size(); ++n) - { - for (size_t m = 0; m < pubkeys[n].size(); ++m) - { - if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[n][m].dest)) - { - MERROR_VER("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m); - return false; - } - if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[n][m].mask)) - { - MERROR_VER("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m); - return false; - } - } - } - } - - const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size(); - if (n_sigs != tx.vin.size()) - { - MERROR_VER("Failed to check ringct signatures: mismatched MGs/vin sizes"); - return false; - } - for (size_t n = 0; n < tx.vin.size(); ++n) - { - bool error; - if (rct::is_rct_clsag(rv.type)) - error = memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.CLSAGs[n].I, 32); - else - error = rv.p.MGs[n].II.empty() || memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.MGs[n].II[0], 32); - if (error) - { - MERROR_VER("Failed to check ringct signatures: mismatched key image"); - return false; - } - } - - if (!rct::verRctNonSemanticsSimple(rv)) + if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, RCT_CACHE_TYPE)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -3622,6 +3589,12 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } case rct::RCTTypeFull: { + if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys)) + { + MERROR_VER("Failed to expand rct signatures!"); + return false; + } + // check all this, either reconstructed (so should really pass), or not { bool size_matches = true; @@ -4552,11 +4525,15 @@ leave: } } - send_miner_notifications(id, already_generated_coins); + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height)); + send_miner_notifications(new_height, seedhash, id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + return true; } //------------------------------------------------------------------ @@ -5761,24 +5738,15 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_valid = true; } -void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins) +void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins) { if (m_miner_notifiers.empty()) return; - const uint64_t height = m_db->height(); const uint8_t major_version = m_hardfork->get_ideal_version(height); const difficulty_type diff = get_difficulty_for_next_block(); const uint64_t median_weight = m_current_block_cumul_weight_median; - crypto::hash seed_hash = crypto::null_hash; - if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) - { - uint64_t seed_height, next_height; - crypto::rx_seedheights(height, &seed_height, &next_height); - seed_hash = get_block_id_by_height(seed_height); - } - std::vector<tx_block_template_backlog_entry> tx_backlog; m_tx_pool.get_block_template_backlog(tx_backlog); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 4795fc55c..42246fca2 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -57,6 +57,7 @@ #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_basic/difficulty.h" #include "cryptonote_tx_utils.h" +#include "tx_verification_utils.h" #include "cryptonote_basic/verification_context.h" #include "crypto/hash.h" #include "checkpoints/checkpoints.h" @@ -597,6 +598,15 @@ namespace cryptonote bool store_blockchain(); /** + * @brief expands v2 transaction data from blockchain + * + * RingCT transactions do not transmit some of their data if it + * can be reconstituted by the receiver. This function expands + * that implicit data. + */ + static bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys); + + /** * @brief validates a transaction's inputs * * validates a transaction's inputs as correctly used and not previously @@ -1222,6 +1232,9 @@ namespace cryptonote uint64_t m_prepare_nblocks; std::vector<block> *m_prepare_blocks; + // cache for verifying transaction RCT non semantics + mutable rct_ver_cache_t m_rct_ver_cache; + /** * @brief collects the keys for all outputs being "spent" as an input * @@ -1575,15 +1588,6 @@ namespace cryptonote void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints); /** - * @brief expands v2 transaction data from blockchain - * - * RingCT transactions do not transmit some of their data if it - * can be reconstituted by the receiver. This function expands - * that implicit data. - */ - bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const; - - /** * @brief invalidates any cached block template */ void invalidate_block_template_cache(); @@ -1598,9 +1602,11 @@ namespace cryptonote /** * @brief sends new block notifications to ZMQ `miner_data` subscribers * + * @param height current blockchain height + * @param seed_hash seed hash to use for mining * @param prev_id hash of new blockchain tip * @param already_generated_coins total coins mined by the network so far */ - void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); + void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d8c782f78..d34c92723 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1099,7 +1099,7 @@ namespace cryptonote else if(tvc[i].m_verifivation_impossible) {MERROR_VER("Transaction verification impossible: " << results[i].hash);} - if(tvc[i].m_added_to_pool) + if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE) { MDEBUG("tx added: " << results[i].hash); valid_events = true; @@ -1727,6 +1727,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const + { + return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes); + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); @@ -1739,6 +1744,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const + { + return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental); + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const { m_mempool.get_transaction_stats(stats, include_sensitive_data); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 6dc513570..efab56405 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -510,6 +510,23 @@ namespace cryptonote bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const; /** + * @copydoc tx_memory_pool::get_pool_transactions_info + * @param include_sensitive_txes include private transactions + * + * @note see tx_memory_pool::get_pool_transactions_info + */ + bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const; + + /** + * @copydoc tx_memory_pool::get_pool_info + * @param include_sensitive_txes include private transactions + * @param max_tx_count max allowed added_txs in response + * + * @note see tx_memory_pool::get_pool_info + */ + bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const; + + /** * @copydoc tx_memory_pool::get_transactions * @param include_sensitive_txes include private transactions * diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 472026217..5058b89a9 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -437,6 +437,8 @@ namespace cryptonote if (!sort_tx_extra(tx.extra, tx.extra)) return false; + CHECK_AND_ASSERT_MES(tx.extra.size() <= MAX_TX_EXTRA_SIZE, false, "TX extra size (" << tx.extra.size() << ") is greater than max allowed (" << MAX_TX_EXTRA_SIZE << ")"); + //check money if(summary_outs_money > summary_inputs_money ) { @@ -669,10 +671,10 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash) + void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash) { blobdata bd = get_block_hashing_blob(b); - rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1); + rx_slow_hash(seed_hash.data, bd.data(), bd.size(), res.data); } bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners) @@ -686,20 +688,16 @@ namespace cryptonote } if (major_version >= RX_BLOCK_VERSION) { - uint64_t seed_height, main_height; crypto::hash hash; if (pbc != NULL) { - seed_height = rx_seedheight(height); + const uint64_t seed_height = rx_seedheight(height); hash = seed_hash ? *seed_hash : pbc->get_pending_block_id_by_height(seed_height); - main_height = pbc->get_current_blockchain_height(); } else { memset(&hash, 0, sizeof(hash)); // only happens when generating genesis block - seed_height = 0; - main_height = 0; } - rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash); + rx_slow_hash(hash.data, bd.data(), bd.size(), res.data); } else { const int pow_variant = major_version >= 7 ? major_version - 6 : 0; crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height); @@ -713,20 +711,10 @@ namespace cryptonote return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners); } - bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners) - { - return get_block_longhash(pbc, b, res, height, NULL, miners); - } - - crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners) + crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const crypto::hash *seed_hash, const int miners) { crypto::hash p = crypto::null_hash; - get_block_longhash(pbc, b, p, height, miners); + get_block_longhash(pbc, b, p, height, seed_hash, miners); return p; } - - void get_block_longhash_reorg(const uint64_t split_height) - { - rx_reorg(split_height); - } } diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 12d6b8ce5..5f301bb89 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -144,14 +144,10 @@ namespace cryptonote ); class Blockchain; - bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, - const int major_version, const crypto::hash *seed_hash, const int miners); - bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners); - bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners); - void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, - const uint64_t seed_height, const crypto::hash& seed_hash); - crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners); - void get_block_longhash_reorg(const uint64_t split_height); + bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners = 0); + bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0); + crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0); + void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash); } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 2a514ceae..74f36f89b 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -40,7 +40,6 @@ #include "blockchain.h" #include "blockchain_db/locked_txn.h" #include "blockchain_db/blockchain_db.h" -#include "common/boost_serialization_helper.h" #include "int-util.h" #include "misc_language.h" #include "warnings.h" @@ -133,6 +132,12 @@ namespace cryptonote // class code expects unsigned values throughout if (m_next_check < time_t(0)) throw std::runtime_error{"Unexpected time_t (system clock) value"}; + + m_added_txs_start_time = (time_t)0; + m_removed_txs_start_time = (time_t)0; + // We don't set these to "now" already here as we don't know how long it takes from construction + // of the pool until it "goes to work". It's safer to set when the first actual txs enter the + // corresponding lists. } //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) @@ -207,6 +212,7 @@ namespace cryptonote { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; + tvc.m_no_drop_offense = true; return false; } @@ -219,6 +225,16 @@ namespace cryptonote return false; } + size_t tx_extra_size = tx.extra.size(); + if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE) + { + LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE); + tvc.m_verifivation_failed = true; + tvc.m_tx_extra_too_big = true; + tvc.m_no_drop_offense = true; + return false; + } + // if the transaction came from a block popped from the chain, // don't check if we have its key images as spent. // TODO: Investigate why not? @@ -281,7 +297,7 @@ namespace cryptonote return false; m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time); lock.commit(); } catch (const std::exception &e) @@ -352,7 +368,7 @@ namespace cryptonote m_blockchain.remove_txpool_tx(id); m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time); } lock.commit(); } @@ -373,7 +389,7 @@ namespace cryptonote ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1))); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size()); prune(m_txpool_max_weight); @@ -464,7 +480,8 @@ namespace cryptonote reduce_txpool_weight(meta.weight); remove_transaction_keyimages(tx, txid); MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); - m_txs_by_fee_and_receive_time.erase(it--); + remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted)); + it--; changed = true; } catch (const std::exception &e) @@ -546,8 +563,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - auto sorted_it = find_tx_in_sorted_container(id); - + bool sensitive = false; try { LockedTXN lock(m_blockchain.get_db()); @@ -578,6 +594,7 @@ namespace cryptonote do_not_relay = meta.do_not_relay; double_spend_seen = meta.double_spend_seen; pruned = meta.pruned; + sensitive = !meta.matches(relay_category::broadcasted); // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); @@ -591,13 +608,12 @@ namespace cryptonote return false; } - if (sorted_it != m_txs_by_fee_and_receive_time.end()) - m_txs_by_fee_and_receive_time.erase(sorted_it); + remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive); ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const + bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const { PERF_TIMER(get_transaction_info); CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -609,7 +625,12 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx in txpool"); + LOG_PRINT_L2("Failed to find tx in txpool: " << txid); + return false; + } + if (!include_sensitive_data && !meta.matches(relay_category::broadcasted)) + { + // We don't want sensitive data && the tx is sensitive, so no need to return it return false; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); @@ -635,11 +656,13 @@ namespace cryptonote td.kept_by_block = meta.kept_by_block; td.last_failed_height = meta.last_failed_height; td.last_failed_id = meta.last_failed_id; - td.receive_time = meta.receive_time; - td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time; + td.receive_time = include_sensitive_data ? meta.receive_time : 0; + td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0; td.relayed = meta.relayed; td.do_not_relay = meta.do_not_relay; td.double_spend_seen = meta.double_spend_seen; + if (include_blob) + td.tx_blob = std::move(txblob); } catch (const std::exception &e) { @@ -649,6 +672,25 @@ namespace cryptonote return true; } + //------------------------------------------------------------------ + bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + txs.clear(); + + for (const auto &it: txids) + { + tx_details details; + bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/); + if (success) + { + txs.push_back(std::make_pair(it, std::move(details))); + } + } + return true; + } //--------------------------------------------------------------------------------- bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const { @@ -710,15 +752,7 @@ namespace cryptonote (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) ) { LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age ); - auto sorted_it = find_tx_in_sorted_container(txid); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - { - LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); - } - else - { - m_txs_by_fee_and_receive_time.erase(sorted_it); - } + remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted)); m_timed_out_transactions.insert(txid); remove.push_back(std::make_pair(txid, meta.weight)); } @@ -872,9 +906,12 @@ namespace cryptonote meta.last_relayed_time = std::chrono::system_clock::to_time_t(now); m_blockchain.update_txpool_tx(hash, meta); - // wait until db update succeeds to ensure tx is visible in the pool was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted); + + if (was_just_broadcasted) + // Make sure the tx gets re-added with an updated time + add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now)); } } catch (const std::exception &e) @@ -927,6 +964,81 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + incremental = true; + if (start_time == (time_t)0) + { + // Giving no start time means give back whole pool + incremental = false; + } + else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0)) + { + if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time)) + { + // If either of the two lists do not go back far enough it's not possible to + // deliver incremental pool info + incremental = false; + } + // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time + } + else + { + // Some incremental info still missing completely + incremental = false; + } + + added_txs.clear(); + remaining_added_txids.clear(); + removed_txs.clear(); + + std::vector<crypto::hash> txids; + if (!incremental) + { + LOG_PRINT_L2("Giving back the whole pool"); + // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the + // anonymous method somehow results in an LMDB error with transactions we have to build a list of + // ids first and get the full info afterwards + get_transaction_hashes(txids, include_sensitive); + if (txids.size() > max_tx_count) + { + remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end()); + txids.erase(txids.begin() + max_tx_count, txids.end()); + } + get_transactions_info(txids, added_txs, include_sensitive); + return true; + } + + // Give back incrementally, based on time of entry into the map + for (const auto &pit : m_added_txs_by_id) + { + if (pit.second >= start_time) + txids.push_back(pit.first); + } + get_transactions_info(txids, added_txs, include_sensitive); + if (added_txs.size() > max_tx_count) + { + remaining_added_txids.reserve(added_txs.size() - max_tx_count); + for (size_t i = max_tx_count; i < added_txs.size(); ++i) + remaining_added_txids.push_back(added_txs[i].first); + added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end()); + } + + std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time); + while (rit != m_removed_txs_by_time.end()) + { + if (include_sensitive || !rit->second.sensitive) + { + removed_txs.push_back(rit->second.txid); + } + ++rit; + } + return true; + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1631,6 +1743,12 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + // Simply throw away incremental info, too difficult to update + m_added_txs_by_id.clear(); + m_added_txs_start_time = (time_t)0; + m_removed_txs_by_time.clear(); + m_removed_txs_start_time = (time_t)0; + MINFO("Validating txpool contents for v" << (unsigned)version); LockedTXN lock(m_blockchain.get_db()); @@ -1688,6 +1806,106 @@ namespace cryptonote return n_removed; } //--------------------------------------------------------------------------------- + void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time) + { + + time_t now = time(NULL); + const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid); + if (it == m_added_txs_by_id.end()) + { + m_added_txs_by_id.insert(std::make_pair(txid, now)); + } + else + { + // This tx was already added to the map earlier, probably because then it was in the "stem" + // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set + // a new time for clients that are not allowed to see sensitive txs to make sure they will + // see it now if they query incrementally + it->second = now; + + auto sorted_it = find_tx_in_sorted_container(txid); + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + MERROR("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + } + m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid); + + // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time' + // whether we have that txid there and if yes remove it; this results in possible duplicates + // where we return certain txids as deleted AND in the pool at the same time which requires + // clients to process deleted ones BEFORE processing pool txs + if (m_added_txs_start_time == (time_t)0) + { + m_added_txs_start_time = now; + } + } + //--------------------------------------------------------------------------------- + void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive) + { + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + + const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid); + if (it != m_added_txs_by_id.end()) + { + m_added_txs_by_id.erase(it); + } + else + { + MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs"); + } + track_removed_tx(txid, sensitive); + } + //--------------------------------------------------------------------------------- + void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive) + { + time_t now = time(NULL); + m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive})); + MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size()); + if (m_removed_txs_start_time == (time_t)0) + { + m_removed_txs_start_time = now; + } + + // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set + // an absolute size limit plus delete entries that are x minutes old (which is ok because clients + // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back) + const int MAX_REMOVED = 20000; + if (m_removed_txs_by_time.size() > MAX_REMOVED) + { + auto erase_it = m_removed_txs_by_time.begin(); + std::advance(erase_it, MAX_REMOVED / 4 + 1); + m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it); + m_removed_txs_start_time = m_removed_txs_by_time.begin()->first; + MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size()); + } + else + { + time_t earliest = now - (30 * 60); // 30 minutes + std::map<time_t, removed_tx_info>::iterator from, to; + from = m_removed_txs_by_time.begin(); + to = m_removed_txs_by_time.lower_bound(earliest); + int distance = std::distance(from, to); + if (distance > 0) + { + m_removed_txs_by_time.erase(from, to); + m_removed_txs_start_time = earliest; + MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size()); + } + } + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1695,6 +1913,10 @@ namespace cryptonote m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); + m_added_txs_by_id.clear(); + m_added_txs_start_time = (time_t)0; + m_removed_txs_by_time.clear(); + m_removed_txs_start_time = (time_t)0; m_spent_key_images.clear(); m_txpool_weight = 0; std::vector<crypto::hash> remove; @@ -1719,7 +1941,7 @@ namespace cryptonote MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); + add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time); m_txpool_weight += meta.weight; return true; }, true, relay_category::all); diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 45623fd14..23135ead1 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -428,6 +428,7 @@ namespace cryptonote struct tx_details { transaction tx; //!< the transaction + cryptonote::blobdata tx_blob; //!< the transaction's binary blob size_t blob_size; //!< the transaction's size size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount @@ -466,13 +467,25 @@ namespace cryptonote /** * @brief get infornation about a single transaction */ - bool get_transaction_info(const crypto::hash &txid, tx_details &td) const; + bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const; + + /** + * @brief get information about multiple transactions + */ + bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const; /** * @brief get transactions not in the passed set */ bool get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const; + /** + * @brief get info necessary for update of pool-related info in a wallet, preferably incremental + * + * @return true on success, false on error + */ + bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const; + private: /** @@ -577,6 +590,10 @@ namespace cryptonote */ void prune(size_t bytes = 0); + void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time); + void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive); + void track_removed_tx(const crypto::hash& txid, bool sensitive); + //TODO: confirm the below comments and investigate whether or not this // is the desired behavior //! map key images to transactions which spent them @@ -609,6 +626,26 @@ private: std::atomic<uint64_t> m_cookie; //!< incremented at each change + // Info when transactions entered the pool, accessible by txid + std::unordered_map<crypto::hash, time_t> m_added_txs_by_id; + + // Info at what time the pool started to track the adding of transactions + time_t m_added_txs_start_time; + + struct removed_tx_info + { + crypto::hash txid; + bool sensitive; + }; + + // Info about transactions that were removed from the pool, ordered by the time + // of deletion + std::multimap<time_t, removed_tx_info> m_removed_txs_by_time; + + // Info how far back in time the list of removed tx ids currently reaches + // (it gets shorted periodically to prevent overflow) + time_t m_removed_txs_start_time; + /** * @brief get an iterator to a transaction in the sorted container * diff --git a/src/cryptonote_core/tx_verification_utils.cpp b/src/cryptonote_core/tx_verification_utils.cpp new file mode 100644 index 000000000..a93ef2f25 --- /dev/null +++ b/src/cryptonote_core/tx_verification_utils.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/tx_verification_utils.h" +#include "ringct/rctSigs.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "blockchain" + +#define VER_ASSERT(cond, msgexpr) CHECK_AND_ASSERT_MES(cond, false, msgexpr) + +using namespace cryptonote; + +// Do RCT expansion, then do post-expansion sanity checks, then do full non-semantics verification. +static bool expand_tx_and_ver_rct_non_sem(transaction& tx, const rct::ctkeyM& mix_ring) +{ + // Pruned transactions can not be expanded and verified because they are missing RCT data + VER_ASSERT(!tx.pruned, "Pruned transaction will not pass verRctNonSemanticsSimple"); + + // Calculate prefix hash + const crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); + + // Expand mixring, tx inputs, tx key images, prefix hash message, etc into the RCT sig + const bool exp_res = Blockchain::expand_transaction_2(tx, tx_prefix_hash, mix_ring); + VER_ASSERT(exp_res, "Failed to expand rct signatures!"); + + const rct::rctSig& rv = tx.rct_signatures; + + // Check that expanded RCT mixring == input mixring + VER_ASSERT(rv.mixRing == mix_ring, "Failed to check ringct signatures: mismatched pubkeys/mixRing"); + + // Check CLSAG/MLSAG size against transaction input + const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size(); + VER_ASSERT(n_sigs == tx.vin.size(), "Failed to check ringct signatures: mismatched input sigs/vin sizes"); + + // For each input, check that the key images were copied into the expanded RCT sig correctly + for (size_t n = 0; n < n_sigs; ++n) + { + const crypto::key_image& nth_vin_image = boost::get<txin_to_key>(tx.vin[n]).k_image; + + if (rct::is_rct_clsag(rv.type)) + { + const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.CLSAGs[n].I, 32); + VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched CLSAG key image"); + } + else + { + const bool mg_nonempty = !rv.p.MGs[n].II.empty(); + VER_ASSERT(mg_nonempty, "Failed to check ringct signatures: missing MLSAG key image"); + const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.MGs[n].II[0], 32); + VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched MLSAG key image"); + } + } + + // Mix ring data is now known to be correctly incorporated into the RCT sig inside tx. + return rct::verRctNonSemanticsSimple(rv); +} + +// Create a unique identifier for pair of tx blob + mix ring +static crypto::hash calc_tx_mixring_hash(const transaction& tx, const rct::ctkeyM& mix_ring) +{ + std::stringstream ss; + + // Start with domain seperation + ss << config::HASH_KEY_TXHASH_AND_MIXRING; + + // Then add TX hash + const crypto::hash tx_hash = get_transaction_hash(tx); + ss.write(tx_hash.data, sizeof(crypto::hash)); + + // Then serialize mix ring + binary_archive<true> ar(ss); + ::do_serialize(ar, const_cast<rct::ctkeyM&>(mix_ring)); + + // Calculate hash of TX hash and mix ring blob + crypto::hash tx_and_mixring_hash; + get_blob_hash(ss.str(), tx_and_mixring_hash); + + return tx_and_mixring_hash; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cryptonote +{ + +bool ver_rct_non_semantics_simple_cached +( + transaction& tx, + const rct::ctkeyM& mix_ring, + rct_ver_cache_t& cache, + const std::uint8_t rct_type_to_cache +) +{ + // Hello future Monero dev! If you got this assert, read the following carefully: + // + // For this version of RCT, the way we guaranteed that verification caches do not generate false + // positives (and thus possibly enabling double spends) is we take a hash of two things. One, + // we use get_transaction_hash() which gives us a (cryptographically secure) unique + // representation of all "knobs" controlled by the possibly malicious constructor of the + // transaction. Two, we take a hash of all *previously validated* blockchain data referenced by + // this transaction which is required to validate the ring signature. In our case, this is the + // mixring. Future versions of the protocol may differ in this regard, but if this assumptions + // holds true in the future, enable the verification hash by modifying the `untested_tx` + // condition below. + const bool untested_tx = tx.version > 2 || tx.rct_signatures.type > rct::RCTTypeBulletproofPlus; + VER_ASSERT(!untested_tx, "Unknown TX type. Make sure RCT cache works correctly with this type and then enable it in the code here."); + + // Don't cache older (or newer) rctSig types + // This cache only makes sense when it caches data from mempool first, + // so only "current fork version-enabled" RCT types need to be cached + if (tx.rct_signatures.type != rct_type_to_cache) + { + MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " skipped"); + return expand_tx_and_ver_rct_non_sem(tx, mix_ring); + } + + // Generate unique hash for tx+mix_ring pair + const crypto::hash tx_mixring_hash = calc_tx_mixring_hash(tx, mix_ring); + + // Search cache for successful verification of same TX + mix ring combination + if (cache.has(tx_mixring_hash)) + { + MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " hit"); + return true; + } + + // We had a cache miss, so now we must expand the mix ring and do full verification + MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " missed"); + if (!expand_tx_and_ver_rct_non_sem(tx, mix_ring)) + { + return false; + } + + // At this point, the TX RCT verified successfully, so add it to the cache and return true + cache.add(tx_mixring_hash); + + return true; +} + +} // namespace cryptonote diff --git a/src/cryptonote_core/tx_verification_utils.h b/src/cryptonote_core/tx_verification_utils.h new file mode 100644 index 000000000..ccd401d2a --- /dev/null +++ b/src/cryptonote_core/tx_verification_utils.h @@ -0,0 +1,78 @@ +// Copyright (c) 2023, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "common/data_cache.h" +#include "cryptonote_basic/cryptonote_basic.h" + +namespace cryptonote +{ + +// Modifying this value should not affect consensus. You can adjust it for performance needs +static constexpr const size_t RCT_VER_CACHE_SIZE = 8192; + +using rct_ver_cache_t = ::tools::data_cache<::crypto::hash, RCT_VER_CACHE_SIZE>; + +/** + * @brief Cached version of rct::verRctNonSemanticsSimple + * + * This function will not affect how the transaction is serialized and it will never modify the + * transaction prefix. + * + * The reference to tx is mutable since the transaction's ring signatures may be expanded by + * Blockchain::expand_transaction_2. However, on cache hits, the transaction will not be + * expanded. This means that the caller does not need to call expand_transaction_2 on this + * transaction before passing it; the transaction will not successfully verify with "old" RCT data + * if the transaction has been otherwise modified since the last verification. + * + * But, if cryptonote::get_transaction_hash(tx) returns a "stale" hash, this function is not + * guaranteed to work. So make sure that the cryptonote::transaction passed has not had + * modifications to it since the last time its hash was fetched without properly invalidating the + * hashes. + * + * rct_type_to_cache can be any RCT version value as long as rct::verRctNonSemanticsSimple works for + * this RCT version, but for most applications, it doesn't make sense to not make this version + * the "current" RCT version (i.e. the version that transactions in the mempool are). + * + * @param tx transaction which contains RCT signature to verify + * @param mix_ring mixring referenced by this tx. THIS DATA MUST BE PREVIOUSLY VALIDATED + * @param cache saves tx+mixring hashes used to cache calls + * @param rct_type_to_cache Only RCT sigs with version (e.g. RCTTypeBulletproofPlus) will be cached + * @return true when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return true + * @return false when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return false + */ +bool ver_rct_non_semantics_simple_cached +( + transaction& tx, + const rct::ctkeyM& mix_ring, + rct_ver_cache_t& cache, + std::uint8_t rct_type_to_cache +); + +} // namespace cryptonote diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index bd2f8ce85..d72bdbae2 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1020,7 +1020,7 @@ namespace cryptonote for (auto& tx : arg.txs) { tx_verification_context tvc{}; - if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true)) + if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); drop_connection(context, false, false); diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp index 846f6c071..5fcc469e9 100644 --- a/src/daemonizer/windows_service.cpp +++ b/src/daemonizer/windows_service.cpp @@ -196,7 +196,7 @@ bool install_service( , 0 //, GENERIC_EXECUTE | GENERIC_READ , SERVICE_WIN32_OWN_PROCESS - , SERVICE_DEMAND_START + , SERVICE_AUTO_START , SERVICE_ERROR_NORMAL , full_command.c_str() , nullptr diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h index a8e4d3b0e..5a33b2aa7 100644 --- a/src/daemonizer/windows_service_runner.h +++ b/src/daemonizer/windows_service_runner.h @@ -146,9 +146,6 @@ namespace windows { m_handler.run(); on_state_change_request_(SERVICE_CONTROL_STOP); - - // Ensure that the service is uninstalled - uninstall_service(m_name); } static void WINAPI on_state_change_request(DWORD control_code) diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 22a1fd986..79530e277 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -530,6 +530,7 @@ namespace hw { {0x2c97, 0x0001, 0, 0xffa0}, {0x2c97, 0x0004, 0, 0xffa0}, {0x2c97, 0x0005, 0, 0xffa0}, + {0x2c97, 0x0006, 0, 0xffa0}, }; bool device_ledger::connect(void) { diff --git a/src/net/parse.cpp b/src/net/parse.cpp index 1df6175b4..92be492a3 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -38,7 +38,7 @@ namespace net { void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port) { - // require ipv6 address format "[addr:addr:addr:...:addr]:port" + // If IPv6 address format with port "[addr:addr:addr:...:addr]:port" if (address.find(']') != std::string::npos) { host = address.substr(1, address.rfind(']') - 1); @@ -47,6 +47,12 @@ namespace net port = address.substr(address.rfind(':') + 1); } } + // Else if IPv6 address format without port e.g. "addr:addr:addr:...:addr" + else if (std::count(address.begin(), address.end(), ':') >= 2) + { + host = address; + } + // Else IPv4, Tor, I2P address or hostname else { host = address.substr(0, address.rfind(':')); diff --git a/src/net/parse.h b/src/net/parse.h index 648076d7b..6ece931c6 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -38,6 +38,16 @@ namespace net { + /*! + * \brief Takes a valid address string (IP, Tor, I2P, or DNS name) and splits it into host and port + * + * The host of an IPv6 addresses in the format "[x:x:..:x]:port" will have the braces stripped. + * For example, when the address is "[ffff::2023]", host will be set to "ffff::2023". + * + * \param address The address string one wants to split + * \param[out] host The host part of the address string. Is always set. + * \param[out] port The port part of the address string. Is only set when address string contains a port. + */ void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port); /*! diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index f33ce977d..30e3d31b9 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -247,7 +247,23 @@ namespace nodetool if (it == m_blocked_hosts.end()) { m_blocked_hosts[host_str] = limit; - added = true; + + // if the host was already blocked due to being in a blocked subnet, let it be silent + bool matches_blocked_subnet = false; + if (addr.get_type_id() == epee::net_utils::address_type::ipv4) + { + auto ipv4_address = addr.template as<epee::net_utils::ipv4_network_address>(); + for (auto jt = m_blocked_subnets.begin(); jt != m_blocked_subnets.end(); ++jt) + { + if (jt->first.matches(ipv4_address)) + { + matches_blocked_subnet = true; + break; + } + } + } + if (!matches_blocked_subnet) + added = true; } else if (it->second < limit || !add_only) it->second = limit; @@ -317,6 +333,7 @@ namespace nodetool limit = std::numeric_limits<time_t>::max(); else limit = now + seconds; + const bool added = m_blocked_subnets.find(subnet) == m_blocked_subnets.end(); m_blocked_subnets[subnet] = limit; // drop any connection to that subnet. This should only have to look into @@ -349,7 +366,10 @@ namespace nodetool conns.clear(); } - MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked."); + if (added) + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked."); + else + MINFO("Subnet " << subnet.host_str() << " blocked."); return true; } //----------------------------------------------------------------------------------- @@ -645,20 +665,10 @@ namespace nodetool { using namespace boost::asio; - std::string host = addr; + // Split addr string into host string and port string + std::string host; std::string port = std::to_string(default_port); - size_t colon_pos = addr.find_last_of(':'); - size_t dot_pos = addr.find_last_of('.'); - size_t square_brace_pos = addr.find('['); - - // IPv6 will have colons regardless. IPv6 and IPv4 address:port will have a colon but also either a . or a [ - // as IPv6 addresses specified as address:port are to be specified as "[addr:addr:...:addr]:port" - // One may also specify an IPv6 address as simply "[addr:addr:...:addr]" without the port; in that case - // the square braces will be stripped here. - if ((std::string::npos != colon_pos && std::string::npos != dot_pos) || std::string::npos != square_brace_pos) - { - net::get_network_address_host_and_port(addr, host, port); - } + net::get_network_address_host_and_port(addr, host, port); MINFO("Resolving node address: host=" << host << ", port=" << port); io_service io_srv; @@ -695,34 +705,32 @@ namespace nodetool std::set<std::string> full_addrs; if (m_nettype == cryptonote::TESTNET) { - full_addrs.insert("212.83.175.67:28080"); - full_addrs.insert("212.83.172.165:28080"); full_addrs.insert("176.9.0.187:28080"); full_addrs.insert("88.99.173.38:28080"); full_addrs.insert("51.79.173.165:28080"); + full_addrs.insert("192.99.8.110:28080"); + full_addrs.insert("37.187.74.171:28080"); } else if (m_nettype == cryptonote::STAGENET) { - full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("176.9.0.187:38080"); full_addrs.insert("88.99.173.38:38080"); full_addrs.insert("51.79.173.165:38080"); + full_addrs.insert("192.99.8.110:38080"); + full_addrs.insert("37.187.74.171:38080"); } else if (m_nettype == cryptonote::FAKECHAIN) { } else { - full_addrs.insert("212.83.175.67:18080"); - full_addrs.insert("212.83.172.165:18080"); full_addrs.insert("176.9.0.187:18080"); full_addrs.insert("88.198.163.90:18080"); - full_addrs.insert("95.217.25.101:18080"); - full_addrs.insert("136.244.105.131:18080"); - full_addrs.insert("104.238.221.81:18080"); full_addrs.insert("66.85.74.134:18080"); full_addrs.insert("88.99.173.38:18080"); full_addrs.insert("51.79.173.165:18080"); + full_addrs.insert("192.99.8.110:18080"); + full_addrs.insert("37.187.74.171:18080"); } return full_addrs; } @@ -857,6 +865,8 @@ namespace nodetool "4pixvbejrvihnkxmduo2agsnmc3rrulrqc7s3cbwwrep6h6hrzsibeqd.onion:18083", "zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083", "qz43zul2x56jexzoqgkx2trzwcfnr6l3hbtfcfx54g4r3eahy3bssjyd.onion:18083", + "plowsof3t5hogddwabaeiyrno25efmzfxyro2vligremt7sxpsclfaid.onion:18083", + "plowsoffjexmxalw73tkjmf422gq6575fc7vicuu4javzn2ynnte6tyd.onion:18083", }; } return {}; @@ -865,7 +875,9 @@ namespace nodetool { return { "s3l6ke4ed3df466khuebb4poienoingwof7oxtbo6j4n56sghe3a.b32.i2p:18080", - "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080" + "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080", + "uqj3aphckqtjsitz7kxx5flqpwjlq5ppr3chazfued7xucv3nheq.b32.i2p:18080", + "vdmnehdjkpkg57nthgnjfuaqgku673r5bpbqg56ix6fyqoywgqrq.b32.i2p:18080", }; } return {}; @@ -2413,7 +2425,7 @@ namespace nodetool return false; } return true; - }); + }, "0.0.0.0", m_ssl_support); if(!r) { LOG_WARNING_CC(context, "Failed to call connect_async, network error."); diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 67814682a..46950b883 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -57,28 +57,28 @@ extern "C" // 1 1 52.8 70.4 70.2 // Pippenger: -// 1 2 3 4 5 6 7 8 9 bestN -// 2 555 598 621 804 1038 1733 2486 5020 8304 1 -// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 -// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 -// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 -// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 -// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 -// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 -// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 -// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 -// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 -// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 -// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 - -// 2 4 8 16 32 64 128 256 512 1024 2048 4096 -// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 -// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 -// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 -// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 - -// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip -// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip // New timings: // Pippenger: @@ -443,7 +443,7 @@ size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache) return sz; } -rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) +ge_p3 straus_p3(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) { CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); @@ -554,7 +554,13 @@ skipfirst: ge_p1p1_to_p3(&res_p3, &p1); } + return res_p3; +} + +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) +{ rct::key res; + const ge_p3 res_p3 = straus_p3(data, cache, STEP); ge_p3_tobytes(res.bytes, &res_p3); return res; } @@ -571,14 +577,6 @@ size_t get_pippenger_c(size_t N) return 9; } -struct pippenger_cached_data -{ - size_t size; - ge_cached *cached; - pippenger_cached_data(): size(0), cached(NULL) {} - ~pippenger_cached_data() { aligned_free(cached); } -}; - std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset, size_t N) { MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); @@ -586,13 +584,11 @@ std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<Mu if (N == 0) N = data.size() - start_offset; CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); - std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); + std::shared_ptr<pippenger_cached_data> cache = std::make_shared<pippenger_cached_data>(); - cache->size = N; - cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); - CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + cache->resize(N); for (size_t i = 0; i < N; ++i) - ge_p3_to_cached(&cache->cached[i], &data[i+start_offset].point); + ge_p3_to_cached(&(*cache)[i], &data[i+start_offset].point); MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); return cache; @@ -600,14 +596,14 @@ std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<Mu size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache) { - return cache->size * sizeof(*cache->cached); + return cache->size() * sizeof(ge_cached); } -rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t cache_size, size_t c) +ge_p3 pippenger_p3(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t cache_size, size_t c) { if (cache != NULL && cache_size == 0) - cache_size = cache->size; - CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size, "Cache is too small"); + cache_size = cache->size(); + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size(), "Cache is too small"); if (c == 0) c = get_pippenger_c(data.size()); CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); @@ -661,9 +657,9 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< if (buckets_init[bucket]) { if (i < cache_size) - add(buckets[bucket], local_cache->cached[i]); + add(buckets[bucket], (*local_cache)[i]); else - add(buckets[bucket], local_cache_2->cached[i - cache_size]); + add(buckets[bucket], (*local_cache_2)[i - cache_size]); } else { @@ -700,8 +696,14 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< } } + return result; +} + +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, const size_t cache_size, const size_t c) +{ rct::key res; - ge_p3_tobytes(res.bytes, &result); + const ge_p3 result_p3 = pippenger_p3(data, cache, cache_size, c); + ge_p3_tobytes(res.bytes, &result_p3); return res; } diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h index 788516c0b..d9a240637 100644 --- a/src/ringct/multiexp.h +++ b/src/ringct/multiexp.h @@ -35,10 +35,16 @@ #define MULTIEXP_H #include <vector> +extern "C" +{ +#include "crypto/crypto-ops.h" +} #include "crypto/crypto.h" #include "rctTypes.h" #include "misc_log_ex.h" +#include <boost/align/aligned_allocator.hpp> + namespace rct { @@ -55,17 +61,19 @@ struct MultiexpData { }; struct straus_cached_data; -struct pippenger_cached_data; +using pippenger_cached_data = std::vector<ge_cached, boost::alignment::aligned_allocator<ge_cached, 4096>>; rct::key bos_coster_heap_conv(std::vector<MultiexpData> data); rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); +ge_p3 straus_p3(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset = 0, size_t N =0); size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); size_t get_pippenger_c(size_t N); -rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t cache_size = 0, size_t c = 0); +ge_p3 pippenger_p3(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t cache_size = 0, size_t c = 0); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, const size_t cache_size = 0, const size_t c = 0); } diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 245a3f477..5b039d907 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -671,7 +671,7 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - static key ecdhHash(const key &k) + key genAmountEncodingFactor(const key &k) { char data[38]; rct::key hash; @@ -700,7 +700,7 @@ namespace rct { if (v2) { unmasked.mask = zero(); - xor8(unmasked.amount, ecdhHash(sharedSec)); + xor8(unmasked.amount, genAmountEncodingFactor(sharedSec)); } else { @@ -715,7 +715,7 @@ namespace rct { if (v2) { masked.mask = genCommitmentMask(sharedSec); - xor8(masked.amount, ecdhHash(sharedSec)); + xor8(masked.amount, genAmountEncodingFactor(sharedSec)); } else { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 679ed1441..da0013ee1 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -184,6 +184,7 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH + key genAmountEncodingFactor(const key &k); key genCommitmentMask(const key &sk); void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 59ed4d6a6..ab1a26b26 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -97,6 +97,14 @@ namespace rct { struct ctkey { key dest; key mask; //C here if public + + bool operator==(const ctkey &other) const { + return (dest == other.dest) && (mask == other.mask); + } + + bool operator!=(const ctkey &other) const { + return !(*this == other); + } }; typedef std::vector<ctkey> ctkeyV; typedef std::vector<ctkeyV> ctkeyM; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 16bcf2c04..38c382a4c 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -598,88 +598,165 @@ namespace cryptonote CHECK_PAYMENT(req, res, 1); - // quick check for noop - if (!req.block_ids.empty()) - { - uint64_t last_block_height; - crypto::hash last_block_hash; - m_core.get_blockchain_top(last_block_height, last_block_hash); - if (last_block_hash == req.block_ids.front()) - { - res.start_height = 0; - res.current_height = m_core.get_current_blockchain_height(); - res.status = CORE_RPC_STATUS_OK; + res.daemon_time = (uint64_t)time(NULL); + // Always set daemon time, and set it early rather than late, as delivering some incremental pool + // info twice because of slightly overlapping time intervals is no problem, whereas producing gaps + // and never delivering something is + + bool get_blocks = false; + bool get_pool = false; + switch (req.requested_info) + { + case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY: + // Compatibility value 0: Clients that do not set 'requested_info' want blocks, and only blocks + get_blocks = true; + break; + case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL: + get_blocks = true; + get_pool = true; + break; + case COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY: + get_pool = true; + break; + default: + res.status = "Failed, wrong requested info"; return true; - } } - size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; - if (m_rpc_payment) + res.pool_info_extent = COMMAND_RPC_GET_BLOCKS_FAST::NONE; + + if (get_pool) { - max_blocks = res.credits / COST_PER_BLOCK; - if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT) - max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; - if (max_blocks == 0) + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + const bool allow_sensitive = !request_has_rpc_origin || !restricted; + const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits<size_t>::max(); + + bool incremental; + std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> added_pool_txs; + bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental); + if (success) { - res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; + res.added_pool_txs.clear(); + if (m_rpc_payment) + { + CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH); + } + for (const auto &added_pool_tx: added_pool_txs) + { + COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info; + info.tx_hash = added_pool_tx.first; + std::stringstream oss; + binary_archive<true> ar(oss); + bool r = req.prune + ? const_cast<cryptonote::transaction&>(added_pool_tx.second.tx).serialize_base(ar) + : ::serialization::serialize(ar, const_cast<cryptonote::transaction&>(added_pool_tx.second.tx)); + if (!r) + { + res.status = "Failed to serialize transaction"; + return true; + } + info.tx_blob = oss.str(); + info.double_spend_seen = added_pool_tx.second.double_spend_seen; + res.added_pool_txs.push_back(std::move(info)); + } + } + if (success) + { + res.pool_info_extent = incremental ? COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL : COMMAND_RPC_GET_BLOCKS_FAST::FULL; + } + else + { + res.status = "Failed to get pool info"; return true; } } - std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT)) + if (get_blocks) { - res.status = "Failed"; - add_host_fail(ctx); - return true; - } - - CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); - - size_t size = 0, ntxes = 0; - res.blocks.reserve(bs.size()); - res.output_indices.reserve(bs.size()); - for(auto& bd: bs) - { - res.blocks.resize(res.blocks.size()+1); - res.blocks.back().pruned = req.prune; - res.blocks.back().block = bd.first.first; - size += bd.first.first.size(); - res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); - ntxes += bd.second.size(); - res.output_indices.back().indices.reserve(1 + bd.second.size()); - if (req.no_miner_tx) - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - res.blocks.back().txs.reserve(bd.second.size()); - for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + // quick check for noop + if (!req.block_ids.empty()) { - res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash}); - i->second.clear(); - i->second.shrink_to_fit(); - size += res.blocks.back().txs.back().blob.size(); + uint64_t last_block_height; + crypto::hash last_block_hash; + m_core.get_blockchain_top(last_block_height, last_block_hash); + if (last_block_hash == req.block_ids.front()) + { + res.start_height = 0; + res.current_height = last_block_height + 1; + res.status = CORE_RPC_STATUS_OK; + return true; + } } - const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); - if (n_txes_to_lookup > 0) + size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; + if (m_rpc_payment) { - std::vector<std::vector<uint64_t>> indices; - bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); - if (!r) + max_blocks = res.credits / COST_PER_BLOCK; + if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT) + max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; + if (max_blocks == 0) { - res.status = "Failed"; + res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; return true; } - if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + } + + std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs; + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT)) + { + res.status = "Failed"; + add_host_fail(ctx); + return true; + } + + CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); + + size_t size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); + for(auto& bd: bs) + { + res.blocks.resize(res.blocks.size()+1); + res.blocks.back().pruned = req.prune; + res.blocks.back().block = bd.first.first; + size += bd.first.first.size(); + res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); + ntxes += bd.second.size(); + res.output_indices.back().indices.reserve(1 + bd.second.size()); + if (req.no_miner_tx) + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); + res.blocks.back().txs.reserve(bd.second.size()); + for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { - res.status = "Failed"; - return true; + res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash}); + i->second.clear(); + i->second.shrink_to_fit(); + size += res.blocks.back().txs.back().blob.size(); + } + + const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); + if (n_txes_to_lookup > 0) + { + std::vector<std::vector<uint64_t>> indices; + bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); + if (!r) + { + res.status = "Failed"; + return true; + } + if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + { + res.status = "Failed"; + return true; + } + for (size_t i = 0; i < indices.size(); ++i) + res.output_indices.back().indices.push_back({std::move(indices[i])}); } - for (size_t i = 0; i < indices.size(); ++i) - res.output_indices.back().indices.push_back({std::move(indices[i])}); } + MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size); } - MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size); res.status = CORE_RPC_STATUS_OK; return true; } @@ -919,17 +996,16 @@ namespace cryptonote // try the pool for any missing txes size_t found_in_pool = 0; std::unordered_set<crypto::hash> pool_tx_hashes; - std::unordered_map<crypto::hash, tx_info> per_tx_pool_tx_info; + std::unordered_map<crypto::hash, tx_memory_pool::tx_details> per_tx_pool_tx_details; if (!missed_txs.empty()) { - std::vector<tx_info> pool_tx_info; - std::vector<spent_key_image_info> pool_key_image_info; - bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted); + std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> pool_txs; + bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted); if(r) { // sort to match original request std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs; - std::vector<tx_info>::const_iterator i; + std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>::const_iterator i; unsigned txs_processed = 0; for (const crypto::hash &h: vh) { @@ -949,36 +1025,23 @@ namespace cryptonote sorted_txs.push_back(std::move(txs[txs_processed])); ++txs_processed; } - else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) + else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair<crypto::hash, tx_memory_pool::tx_details> &pt) { return h == pt.first; })) != pool_txs.end()) { - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx)) - { - res.status = "Failed to parse and validate tx from blob"; - return true; - } + const tx_memory_pool::tx_details &td = i->second; std::stringstream ss; binary_archive<true> ba(ss); - bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); + bool r = const_cast<cryptonote::transaction&>(td.tx).serialize_base(ba); if (!r) { res.status = "Failed to serialize transaction base"; return true; } const cryptonote::blobdata pruned = ss.str(); - const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx); - sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size()))); + const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx); + sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); - const std::string hash_string = epee::string_tools::pod_to_hex(h); - for (const auto &ti: pool_tx_info) - { - if (ti.id_hash == hash_string) - { - per_tx_pool_tx_info.insert(std::make_pair(h, ti)); - break; - } - } + per_tx_pool_tx_details.insert(std::make_pair(h, td)); ++found_in_pool; } } @@ -1009,7 +1072,17 @@ namespace cryptonote CHECK_AND_ASSERT_MES(tx_hash == std::get<0>(tx), false, "mismatched tx hash"); e.tx_hash = *txhi++; e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); - if (req.split || req.prune || std::get<3>(tx).empty()) + + // coinbase txes do not have signatures to prune, so they appear to be pruned if looking just at prunable data being empty + bool pruned = std::get<3>(tx).empty(); + if (pruned) + { + cryptonote::transaction t; + if (cryptonote::parse_and_validate_tx_base_from_blob(std::get<1>(tx), t) && is_coinbase(t)) + pruned = false; + } + + if (req.split || req.prune || pruned) { // use splitted form with pruned and prunable (filled only when prune=false and the daemon has it), leaving as_hex as empty e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx)); @@ -1074,8 +1147,8 @@ namespace cryptonote { e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max(); e.confirmations = 0; - auto it = per_tx_pool_tx_info.find(tx_hash); - if (it != per_tx_pool_tx_info.end()) + auto it = per_tx_pool_tx_details.find(tx_hash); + if (it != per_tx_pool_tx_details.end()) { e.double_spend_seen = it->second.double_spend_seen; e.relayed = it->second.relayed; @@ -1275,6 +1348,8 @@ namespace cryptonote add_reason(reason, "fee too low"); if ((res.too_few_outputs = tvc.m_too_few_outputs)) add_reason(reason, "too few outputs"); + if ((res.tx_extra_too_big = tvc.m_tx_extra_too_big)) + add_reason(reason, "tx-extra too big"); const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index e1222f6eb..63f74108d 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 11 +#define CORE_RPC_VERSION_MINOR 12 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -162,18 +162,29 @@ namespace cryptonote struct COMMAND_RPC_GET_BLOCKS_FAST { + enum REQUESTED_INFO + { + BLOCKS_ONLY = 0, + BLOCKS_AND_POOL = 1, + POOL_ONLY = 2 + }; + struct request_t: public rpc_access_request_base { + uint8_t requested_info; std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; bool no_miner_tx; + uint64_t pool_info_since; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_request_base) + KV_SERIALIZE_OPT(requested_info, (uint8_t)0) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) KV_SERIALIZE_OPT(no_miner_tx, false) + KV_SERIALIZE_OPT(pool_info_since, (uint64_t)0) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -196,12 +207,37 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; + struct pool_tx_info + { + crypto::hash tx_hash; + blobdata tx_blob; + bool double_spend_seen; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_VAL_POD_AS_BLOB(tx_hash) + KV_SERIALIZE(tx_blob) + KV_SERIALIZE(double_spend_seen) + END_KV_SERIALIZE_MAP() + }; + + enum POOL_INFO_EXTENT + { + NONE = 0, + INCREMENTAL = 1, + FULL = 2 + }; + struct response_t: public rpc_access_response_base { std::vector<block_complete_entry> blocks; uint64_t start_height; uint64_t current_height; std::vector<block_output_indices> output_indices; + uint64_t daemon_time; + uint8_t pool_info_extent; + std::vector<pool_tx_info> added_pool_txs; + std::vector<crypto::hash> remaining_added_pool_txids; + std::vector<crypto::hash> removed_pool_txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_response_base) @@ -209,6 +245,17 @@ namespace cryptonote KV_SERIALIZE(start_height) KV_SERIALIZE(current_height) KV_SERIALIZE(output_indices) + KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0) + KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0) + if (pool_info_extent != POOL_INFO_EXTENT::NONE) + { + KV_SERIALIZE(added_pool_txs) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids) + } + if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL) + { + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids) + } END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -592,6 +639,7 @@ namespace cryptonote bool fee_too_low; bool too_few_outputs; bool sanity_check_failed; + bool tx_extra_too_big; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_response_base) @@ -606,6 +654,7 @@ namespace cryptonote KV_SERIALIZE(fee_too_low) KV_SERIALIZE(too_few_outputs) KV_SERIALIZE(sanity_check_failed) + KV_SERIALIZE(tx_extra_too_big) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 2e05f04fb..aca3e3761 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -236,10 +236,8 @@ namespace cryptonote *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(nonce); if (block.major_version >= RX_BLOCK_VERSION) { - const uint64_t seed_height = is_current ? info.seed_height : info.previous_seed_height; const crypto::hash &seed_hash = is_current ? info.seed_hash : info.previous_seed_hash; - const uint64_t height = cryptonote::get_block_height(block); - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, 0, 0); + crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data); } else { diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index f59af575e..984392b08 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -5894,7 +5894,10 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money); + // For manual refresh don't allow incremental checking of the pool: Because we did not process the txs + // for us in the pool during automatic refresh we could miss some of them if we checked the pool + // incrementally here + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money, true, false); if (reset == ResetSoftKeepKI) { diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 0a9ea8f7b..b7acfbcf0 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -392,4 +392,34 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_payment_info(bool mining, boo return boost::none; } +boost::optional<std::string> NodeRPCProxy::get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f) +{ + const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp + for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE) + { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response resp_t = AUTO_VAL_INIT(resp_t); + + const size_t n_txids = std::min<size_t>(SLICE_SIZE, txids.size() - offset); + for (size_t n = offset; n < (offset + n_txids); ++n) + req_t.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[n])); + MDEBUG("asking for " << req_t.txs_hashes.size() << " transactions"); + req_t.decode_as_json = false; + req_t.prune = true; + + bool r = false; + { + const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; + uint64_t pre_call_credits = m_rpc_payment_state.credits; + req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key); + r = net_utils::invoke_http_json("/gettransactions", req_t, resp_t, m_http_client, rpc_timeout); + if (r && resp_t.status == CORE_RPC_STATUS_OK) + check_rpc_cost(m_rpc_payment_state, "/gettransactions", resp_t.credits, pre_call_credits, resp_t.txs.size() * COST_PER_TX); + } + + f(req_t, resp_t, r); + } + return boost::optional<std::string>(); +} + } diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index e320565ac..47a0ff101 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -59,6 +59,7 @@ public: boost::optional<std::string> get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_blocks, std::vector<uint64_t> &fees); boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask); boost::optional<std::string> get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie); + boost::optional<std::string> get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f); private: template<typename T> void handle_payment_changes(const T &res, std::true_type) { diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 1880d7ea0..ca13468c3 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -39,7 +39,9 @@ #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/algorithm/string/predicate.hpp> +#include <boost/archive/binary_iarchive.hpp> #include <boost/asio/ip/address.hpp> +#include <boost/filesystem/operations.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/preprocessor/stringize.hpp> #include <openssl/evp.h> @@ -64,7 +66,6 @@ using namespace epee; #include "multisig/multisig_account.h" #include "multisig/multisig_kex_msg.h" #include "multisig/multisig_tx_builder_ringct.h" -#include "common/boost_serialization_helper.h" #include "common/command_line.h" #include "common/threadpool.h" #include "int-util.h" @@ -1005,7 +1006,7 @@ gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets, double shap const size_t blocks_to_consider = std::min<size_t>(rct_offsets.size(), blocks_in_a_year); const size_t outputs_to_consider = rct_offsets.back() - (blocks_to_consider < rct_offsets.size() ? rct_offsets[rct_offsets.size() - blocks_to_consider - 1] : 0); begin = rct_offsets.data(); - end = rct_offsets.data() + rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE; + end = rct_offsets.data() + rct_offsets.size() - (std::max(1, CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE) - 1); num_rct_outputs = *(end - 1); THROW_WALLET_EXCEPTION_IF(num_rct_outputs == 0, error::wallet_internal_error, "No rct outputs"); average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / static_cast<double>(outputs_to_consider); // this assumes constant target over the whole rct range @@ -1224,6 +1225,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std m_load_deprecated_formats(false), m_credits_target(0), m_enable_multisig(false), + m_pool_info_query_time(0), m_has_ever_refreshed_from_node(false), m_allow_mismatched_daemon_version(false) { @@ -1348,6 +1350,7 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u m_rpc_payment_state.discrepancy = 0; m_rpc_version = 0; m_node_rpc_proxy.invalidate(); + m_pool_info_query_time = 0; } const std::string address = get_daemon_address(); @@ -2663,7 +2666,83 @@ void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::bl error = !cryptonote::parse_and_validate_block_from_blob(blob, bl, bl_id); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t ¤t_height) +void read_pool_txs(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request &req, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response &res, bool r, const std::vector<crypto::hash> &txids, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &txs) +{ + if (r && res.status == CORE_RPC_STATUS_OK) + { + MDEBUG("Reading pool txs"); + if (res.txs.size() == req.txs_hashes.size()) + { + for (const auto &tx_entry: res.txs) + { + if (tx_entry.in_pool) + { + cryptonote::transaction tx; + cryptonote::blobdata bd; + crypto::hash tx_hash; + + if (get_pruned_tx(tx_entry, tx, tx_hash)) + { + const std::vector<crypto::hash>::const_iterator i = std::find_if(txids.begin(), txids.end(), + [tx_hash](const crypto::hash &e) { return e == tx_hash; }); + if (i != txids.end()) + { + txs.push_back(std::make_tuple(tx, tx_hash, tx_entry.double_spend_seen)); + } + else + { + MERROR("Got txid " << tx_hash << " which we did not ask for"); + } + } + else + { + LOG_PRINT_L0("Failed to parse transaction from daemon"); + } + } + else + { + LOG_PRINT_L1("Transaction from daemon was in pool, but is no more"); + } + } + } + else + { + LOG_PRINT_L0("Expected " << req.txs_hashes.size() << " out of " << txids.size() << " tx(es), got " << res.txs.size()); + } + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed) +{ + std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> added_pool_txs; + added_pool_txs.reserve(res.added_pool_txs.size() + res.remaining_added_pool_txids.size()); + + for (const auto &pool_tx: res.added_pool_txs) + { + cryptonote::transaction tx; + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_base_from_blob(pool_tx.tx_blob, tx), + error::wallet_internal_error, "Failed to validate transaction base from daemon"); + added_pool_txs.push_back(std::make_tuple(tx, pool_tx.tx_hash, pool_tx.double_spend_seen)); + } + + // getblocks.bin may return more added pool transactions than we're allowed to request in restricted mode + if (!res.remaining_added_pool_txids.empty()) + { + // request the remaining txs + m_node_rpc_proxy.get_transactions(res.remaining_added_pool_txids, + [this, &res, &added_pool_txs](const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request &req_t, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response &resp_t, bool r) + { + read_pool_txs(req_t, resp_t, r, res.remaining_added_pool_txids, added_pool_txs); + if (!r || resp_t.status != CORE_RPC_STATUS_OK) + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(resp_t.status)); + } + ); + } + + update_pool_state_from_pool_data(res.pool_info_extent == COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL, res.removed_pool_txids, added_pool_txs, process_txs, refreshed); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t ¤t_height) { cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); @@ -2675,6 +2754,10 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, req.start_height = start_height; req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; + req.requested_info = first ? COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL : COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY; + if (try_incremental) + req.pool_info_since = m_pool_info_query_time; + { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; @@ -2684,16 +2767,36 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" + boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); - check_rpc_cost("/getblocks.bin", res.credits, pre_call_credits, 1 + res.blocks.size() * COST_PER_BLOCK); + uint64_t pool_info_cost = res.added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH; + check_rpc_cost("/getblocks.bin", res.credits, pre_call_credits, 1 + res.blocks.size() * COST_PER_BLOCK + pool_info_cost); } blocks_start_height = res.start_height; blocks = std::move(res.blocks); o_indices = std::move(res.output_indices); current_height = res.current_height; + if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE) + m_pool_info_query_time = res.daemon_time; MDEBUG("Pulled blocks: blocks_start_height " << blocks_start_height << ", count " << blocks.size() - << ", height " << blocks_start_height + blocks.size() << ", node height " << res.current_height); + << ", height " << blocks_start_height + blocks.size() << ", node height " << res.current_height + << ", pool info " << static_cast<unsigned int>(res.pool_info_extent)); + + if (first) + { + if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE) + { + process_pool_info_extent(res, m_process_pool_txs, true); + } + else + { + // If we did not get any pool info, neither incremental nor the whole pool, we probably talk + // to a daemon that does not yet support giving back pool info with the 'getblocks' call, + // and we have to update in the "old way" + update_pool_state_by_pool_query(m_process_pool_txs, true); + } + } + } //---------------------------------------------------------------------------------------------------- void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes) @@ -2943,7 +3046,7 @@ void check_block_hard_fork_version(cryptonote::network_type nettype, uint8_t hf_ daemon_is_outdated = height < start_height || height >= end_height; } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception) +void wallet2::pull_and_parse_next_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception) { error = false; last = false; @@ -2965,7 +3068,7 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks // pull the new blocks std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; uint64_t current_height; - pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices, current_height); + pull_blocks(first, try_incremental, start_height, blocks_start_height, short_chain_history, blocks, o_indices, current_height); THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "Mismatched sizes of blocks and o_indices"); tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); @@ -3029,9 +3132,10 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks } } -void wallet2::remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes) +void wallet2::remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes, bool remove_if_found) { - // remove pool txes to us that aren't in the pool anymore + // remove pool txes to us that aren't in the pool anymore (remove_if_found = false), + // or remove pool txes to us that were reported as removed (remove_if_found = true) std::unordered_multimap<crypto::hash, wallet2::pool_payment_details>::iterator uit = m_unconfirmed_payments.begin(); while (uit != m_unconfirmed_payments.end()) { @@ -3046,9 +3150,9 @@ void wallet2::remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashe } } auto pit = uit++; - if (!found) + if ((!remove_if_found && !found) || (remove_if_found && found)) { - MDEBUG("Removing " << txid << " from unconfirmed payments, not found in pool"); + MDEBUG("Removing " << txid << " from unconfirmed payments"); m_unconfirmed_payments.erase(pit); if (0 != m_callback) m_callback->on_pool_tx_removed(txid); @@ -3057,9 +3161,183 @@ void wallet2::remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashe } //---------------------------------------------------------------------------------------------------- -void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed) +// Code that is common to 'update_pool_state_by_pool_query' and 'update_pool_state_from_pool_data': +// Check wether a tx in the pool is worthy of processing because we did not see it +// yet or because it is "interesting" out of special circumstances +bool wallet2::accept_pool_tx_for_processing(const crypto::hash &txid) +{ + bool txid_found_in_up = false; + for (const auto &up: m_unconfirmed_payments) + { + if (up.second.m_pd.m_tx_hash == txid) + { + txid_found_in_up = true; + break; + } + } + if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end()) + { + // if it's for us, we want to keep track of whether we saw a double spend, so don't bail out + if (!txid_found_in_up) + { + LOG_PRINT_L2("Already seen " << txid << ", and not for us, skipped"); + return false; + } + } + if (!txid_found_in_up) + { + LOG_PRINT_L1("Found new pool tx: " << txid); + bool found = false; + for (const auto &i: m_unconfirmed_txs) + { + if (i.first == txid) + { + found = true; + // if this is a payment to yourself at a different subaddress account, don't skip it + // so that you can see the incoming pool tx with 'show_transfers' on that receiving subaddress account + const unconfirmed_transfer_details& utd = i.second; + for (const auto& dst : utd.m_dests) + { + auto subaddr_index = m_subaddresses.find(dst.addr.m_spend_public_key); + if (subaddr_index != m_subaddresses.end() && subaddr_index->second.major != utd.m_subaddr_account) + { + found = false; + break; + } + } + break; + } + } + if (!found) + { + // not one of those we sent ourselves + return true; + } + else + { + LOG_PRINT_L1("We sent that one"); + return false; + } + } + else { + return false; + } +} +//---------------------------------------------------------------------------------------------------- +// Code that is common to 'update_pool_state_by_pool_query' and 'update_pool_state_from_pool_data': +// Process an unconfirmed transfer after we know whether it's in the pool or not +void wallet2::process_unconfirmed_transfer(bool incremental, const crypto::hash &txid, wallet2::unconfirmed_transfer_details &tx_details, bool seen_in_pool, std::chrono::system_clock::time_point now, bool refreshed) +{ + // TODO: set tx_propagation_timeout to CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE * 3 / 2 after v15 hardfork + constexpr const std::chrono::seconds tx_propagation_timeout{500}; + if (seen_in_pool) + { + if (tx_details.m_state != wallet2::unconfirmed_transfer_details::pending_in_pool) + { + tx_details.m_state = wallet2::unconfirmed_transfer_details::pending_in_pool; + MINFO("Pending txid " << txid << " seen in pool, marking as pending in pool"); + } + } + else + { + if (!incremental) + { + if (tx_details.m_state == wallet2::unconfirmed_transfer_details::pending_in_pool) + { + // For the probably unlikely case that a tx once seen in the pool vanishes + // again set back to 'pending' + tx_details.m_state = wallet2::unconfirmed_transfer_details::pending; + MINFO("Already seen txid " << txid << " vanished from pool, marking as pending"); + } + } + // If a tx is pending for a "long time" without appearing in the pool, and if + // we have refreshed and thus had a chance to really see it if it was there, + // judge it as failed; the waiting for timeout and refresh happened avoids + // false alarms with txs going to 'failed' too early + if (tx_details.m_state == wallet2::unconfirmed_transfer_details::pending && refreshed && + now > std::chrono::system_clock::from_time_t(tx_details.m_sent_time) + tx_propagation_timeout) + { + LOG_PRINT_L1("Pending txid " << txid << " not in pool after " << tx_propagation_timeout.count() << + " seconds, marking as failed"); + tx_details.m_state = wallet2::unconfirmed_transfer_details::failed; + + // the inputs aren't spent anymore, since the tx failed + for (size_t vini = 0; vini < tx_details.m_tx.vin.size(); ++vini) + { + if (tx_details.m_tx.vin[vini].type() == typeid(txin_to_key)) + { + txin_to_key &tx_in_to_key = boost::get<txin_to_key>(tx_details.m_tx.vin[vini]); + for (size_t i = 0; i < m_transfers.size(); ++i) + { + const transfer_details &td = m_transfers[i]; + if (td.m_key_image == tx_in_to_key.k_image) + { + LOG_PRINT_L1("Resetting spent status for output " << vini << ": " << td.m_key_image); + set_unspent(i); + break; + } + } + } + } + } + } +} +//---------------------------------------------------------------------------------------------------- +// This public method is typically called to make sure that the wallet's pool state is up-to-date by +// clients like simplewallet and the RPC daemon. Before incremental update this was the same method +// that 'refresh' also used, but now it's more complicated because for the time being we support +// the "old" and the "new" way of updating the pool and because only the 'getblocks' call supports +// incremental update but we don't want any blocks here. +// +// simplewallet does NOT update the pool info during automatic refresh to avoid disturbing interactive +// messages and prompts. When it finally calls this method here "to catch up" so to say we can't use +// incremental update anymore, because with that we might miss some txs altogether. +void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed, bool try_incremental) +{ + bool updated = false; + if (m_pool_info_query_time != 0 && try_incremental) + { + // We are connected to a daemon that supports giving back pool data with the 'getblocks' call, + // thus use that, to get the chance to work incrementally and to keep working incrementally; + // 'POOL_ONLY' was created to support this case + cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); + + req.requested_info = COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY; + req.pool_info_since = m_pool_info_query_time; + + { + const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; + uint64_t pre_call_credits = m_rpc_payment_state.credits; + req.client = get_client_signature(); + bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, *m_http_client, rpc_timeout); + THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "getblocks.bin", error::get_blocks_error, get_rpc_status(res.status)); + uint64_t pool_info_cost = res.added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH; + check_rpc_cost("/getblocks.bin", res.credits, pre_call_credits, pool_info_cost); + } + + m_pool_info_query_time = res.daemon_time; + if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE) + { + process_pool_info_extent(res, process_txs, refreshed); + updated = true; + } + // We SHOULD get pool data here, but if for some crazy reason we don't fall back to the "old" method + } + if (!updated) + { + update_pool_state_by_pool_query(process_txs, refreshed); + } +} +//---------------------------------------------------------------------------------------------------- +// This is the "old" way of updating the pool with separate queries to get the pool content, used before +// the 'getblocks' command was able to give back pool data in addition to blocks. Before this code was +// the public 'update_pool_state' method. The logic is unchanged. This is a candidate for elimination +// when it's sure that no more "old" daemons can be possibly around. +void wallet2::update_pool_state_by_pool_query(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed) { - MTRACE("update_pool_state start"); + MTRACE("update_pool_state_by_pool_query start"); + process_txs.clear(); auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { m_encrypt_keys_after_refresh.reset(); @@ -3077,16 +3355,15 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_transaction_pool_hashes.bin", error::get_tx_pool_error); check_rpc_cost("/get_transaction_pool_hashes.bin", res.credits, pre_call_credits, 1 + res.tx_hashes.size() * COST_PER_POOL_HASH); } - MTRACE("update_pool_state got pool"); + MTRACE("update_pool_state_by_pool_query got pool"); // remove any pending tx that's not in the pool - // TODO: set tx_propagation_timeout to CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE * 3 / 2 after v15 hardfork - constexpr const std::chrono::seconds tx_propagation_timeout{500}; const auto now = std::chrono::system_clock::now(); std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin(); while (it != m_unconfirmed_txs.end()) { const crypto::hash &txid = it->first; + MDEBUG("Checking m_unconfirmed_txs entry " << txid); bool found = false; for (const auto &it2: res.tx_hashes) { @@ -3097,193 +3374,115 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, } } auto pit = it++; - if (!found) - { - // we want to avoid a false positive when we ask for the pool just after - // a tx is removed from the pool due to being found in a new block, but - // just before the block is visible by refresh. So we keep a boolean, so - // that the first time we don't see the tx, we set that boolean, and only - // delete it the second time it is checked (but only when refreshed, so - // we're sure we've seen the blockchain state first) - if (pit->second.m_state == wallet2::unconfirmed_transfer_details::pending) - { - LOG_PRINT_L1("Pending txid " << txid << " not in pool, marking as not in pool"); - pit->second.m_state = wallet2::unconfirmed_transfer_details::pending_not_in_pool; - } - else if (pit->second.m_state == wallet2::unconfirmed_transfer_details::pending_not_in_pool && refreshed && - now > std::chrono::system_clock::from_time_t(pit->second.m_sent_time) + tx_propagation_timeout) - { - LOG_PRINT_L1("Pending txid " << txid << " not in pool after " << tx_propagation_timeout.count() << - " seconds, marking as failed"); - pit->second.m_state = wallet2::unconfirmed_transfer_details::failed; - - // the inputs aren't spent anymore, since the tx failed - for (size_t vini = 0; vini < pit->second.m_tx.vin.size(); ++vini) - { - if (pit->second.m_tx.vin[vini].type() == typeid(txin_to_key)) - { - txin_to_key &tx_in_to_key = boost::get<txin_to_key>(pit->second.m_tx.vin[vini]); - for (size_t i = 0; i < m_transfers.size(); ++i) - { - const transfer_details &td = m_transfers[i]; - if (td.m_key_image == tx_in_to_key.k_image) - { - LOG_PRINT_L1("Resetting spent status for output " << vini << ": " << td.m_key_image); - set_unspent(i); - break; - } - } - } - } - } - } + process_unconfirmed_transfer(false, txid, pit->second, found, now, refreshed); + MDEBUG("New state of that entry: " << pit->second.m_state); } - MTRACE("update_pool_state done first loop"); + MTRACE("update_pool_state_by_pool_query done first loop"); // remove pool txes to us that aren't in the pool anymore // but only if we just refreshed, so that the tx can go in // the in transfers list instead (or nowhere if it just // disappeared without being mined) if (refreshed) - remove_obsolete_pool_txs(res.tx_hashes); + remove_obsolete_pool_txs(res.tx_hashes, false); - MTRACE("update_pool_state done second loop"); + MTRACE("update_pool_state_by_pool_query done second loop"); // gather txids of new pool txes to us - std::vector<std::pair<crypto::hash, bool>> txids; + std::vector<crypto::hash> txids; for (const auto &txid: res.tx_hashes) { - bool txid_found_in_up = false; - for (const auto &up: m_unconfirmed_payments) + if (accept_pool_tx_for_processing(txid)) + txids.push_back(txid); + } + + m_node_rpc_proxy.get_transactions(txids, + [this, &txids, &process_txs](const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request &req_t, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response &resp_t, bool r) { - if (up.second.m_pd.m_tx_hash == txid) - { - txid_found_in_up = true; - break; - } + read_pool_txs(req_t, resp_t, r, txids, process_txs); + if (!r || resp_t.status != CORE_RPC_STATUS_OK) + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(resp_t.status)); } - if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end()) + ); + + MTRACE("update_pool_state_by_pool_query end"); +} +//---------------------------------------------------------------------------------------------------- +// Update pool state from pool data we got together with block data, either incremental data with +// txs that are new in the pool since the last time we queried and the ids of txs that were +// removed from the pool since then, or the whole content of the pool if incremental was not +// possible, e.g. because the server was just started or restarted. +void wallet2::update_pool_state_from_pool_data(bool incremental, const std::vector<crypto::hash> &removed_pool_txids, const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &added_pool_txs, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed) +{ + MTRACE("update_pool_state_from_pool_data start"); + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + m_encrypt_keys_after_refresh.reset(); + }); + + if (refreshed) + { + if (incremental) { - // if it's for us, we want to keep track of whether we saw a double spend, so don't bail out - if (!txid_found_in_up) - { - LOG_PRINT_L2("Already seen " << txid << ", and not for us, skipped"); - continue; - } + // Delete from the list of unconfirmed payments what the daemon reported as tx that was removed from + // pool; do so only after refresh to not delete too early and too eagerly; maybe we will find the tx + // later in a block, or not, or find it again in the pool txs because it was first removed but then + // somehow quickly "resurrected" - that all does not matter here, we retrace the removal + remove_obsolete_pool_txs(removed_pool_txids, true); } - if (!txid_found_in_up) + else { - LOG_PRINT_L1("Found new pool tx: " << txid); - bool found = false; - for (const auto &i: m_unconfirmed_txs) + // Delete from the list of unconfirmed payments what we don't find anymore in the pool; a bit + // unfortunate that we have to build a new vector with ids first, but better than copying and + // modifying the code of 'remove_obsolete_pool_txs' here + std::vector<crypto::hash> txids; + txids.reserve(added_pool_txs.size()); + for (const auto &pool_tx: added_pool_txs) { - if (i.first == txid) - { - found = true; - // if this is a payment to yourself at a different subaddress account, don't skip it - // so that you can see the incoming pool tx with 'show_transfers' on that receiving subaddress account - const unconfirmed_transfer_details& utd = i.second; - for (const auto& dst : utd.m_dests) - { - auto subaddr_index = m_subaddresses.find(dst.addr.m_spend_public_key); - if (subaddr_index != m_subaddresses.end() && subaddr_index->second.major != utd.m_subaddr_account) - { - found = false; - break; - } - } - break; - } - } - if (!found) - { - // not one of those we sent ourselves - txids.push_back({txid, false}); - } - else - { - LOG_PRINT_L1("We sent that one"); + txids.push_back(std::get<1>(pool_tx)); } + remove_obsolete_pool_txs(txids, false); } } - // get_transaction_pool_hashes.bin may return more transactions than we're allowed to request in restricted mode - const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp - for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE) + // Possibly remove any pending tx that's not in the pool + const auto now = std::chrono::system_clock::now(); + std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin(); + while (it != m_unconfirmed_txs.end()) { - cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; - cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res; - - const size_t n_txids = std::min<size_t>(SLICE_SIZE, txids.size() - offset); - for (size_t n = offset; n < (offset + n_txids); ++n) { - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids.at(n).first)); - } - MDEBUG("asking for " << req.txs_hashes.size() << " transactions"); - req.decode_as_json = false; - req.prune = true; - - bool r; - { - const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - uint64_t pre_call_credits = m_rpc_payment_state.credits; - req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); - if (r && res.status == CORE_RPC_STATUS_OK) - check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX); - } - - MDEBUG("Got " << r << " and " << res.status); - if (r && res.status == CORE_RPC_STATUS_OK) + const crypto::hash &txid = it->first; + MDEBUG("Checking m_unconfirmed_txs entry " << txid); + bool found = false; + for (const auto &pool_tx: added_pool_txs) { - if (res.txs.size() == req.txs_hashes.size()) + if (std::get<1>(pool_tx) == txid) { - for (const auto &tx_entry: res.txs) - { - if (tx_entry.in_pool) - { - cryptonote::transaction tx; - cryptonote::blobdata bd; - crypto::hash tx_hash; - - if (get_pruned_tx(tx_entry, tx, tx_hash)) - { - const std::vector<std::pair<crypto::hash, bool>>::const_iterator i = std::find_if(txids.begin(), txids.end(), - [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); - if (i != txids.end()) - { - process_txs.push_back(std::make_tuple(tx, tx_hash, tx_entry.double_spend_seen)); - } - else - { - MERROR("Got txid " << tx_hash << " which we did not ask for"); - } - } - else - { - LOG_PRINT_L0("Failed to parse transaction from daemon"); - } - } - else - { - LOG_PRINT_L1("Transaction from daemon was in pool, but is no more"); - } - } - } - else - { - LOG_PRINT_L0("Expected " << n_txids << " out of " << txids.size() << " tx(es), got " << res.txs.size()); + found = true; + break; } } - else + auto pit = it++; + process_unconfirmed_transfer(incremental, txid, pit->second, found, now, refreshed); + MDEBUG("Resulting state of that entry: " << pit->second.m_state); + } + + // Collect all pool txs that are "interesting" i.e. mostly those that we don't know about yet; + // if we work incrementally and thus see only new pool txs since last time we asked it should + // be rare that we know already about one of those, but check nevertheless + process_txs.clear(); + for (const auto &pool_tx: added_pool_txs) + { + if (accept_pool_tx_for_processing(std::get<1>(pool_tx))) { - LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(res.status)); + process_txs.push_back(pool_tx); } } - MTRACE("update_pool_state end"); + + MTRACE("update_pool_state_from_pool_data end"); } //---------------------------------------------------------------------------------------------------- void wallet2::process_pool_state(const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &txs) { + MTRACE("process_pool_state start"); const time_t now = time(NULL); for (const auto &e: txs) { @@ -3298,6 +3497,7 @@ void wallet2::process_pool_state(const std::vector<std::tuple<cryptonote::transa m_scanned_pool_txs[0].clear(); } } + MTRACE("process_pool_state end"); } //---------------------------------------------------------------------------------------------------- void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force) @@ -3418,7 +3618,7 @@ std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> wallet2::create return cache; } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool, uint64_t max_blocks) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool, bool try_incremental, uint64_t max_blocks) { if (m_offline) { @@ -3506,12 +3706,15 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo auto scope_exit_handler_hwdev = epee::misc_utils::create_scope_leave_handler([&](){hwdev.computing_key_images(false);}); + m_process_pool_txs.clear(); + // Getting and processing the pool state has moved down into method 'pull_blocks' to + // allow for "conventional" as well as "incremental" update. However the following + // principle of getting all info first (pool AND blocks) and only process txs afterwards + // still holds and is still respected: // get updated pool state first, but do not process those txes just yet, // since that might cause a password prompt, which would introduce a data // leak allowing a passive adversary with traffic analysis capability to // infer when we get an incoming output - std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> process_pool_txs; - update_pool_state(process_pool_txs, true); bool first = true, last = false; while(m_run.load(std::memory_order_relaxed) && blocks_fetched < max_blocks) @@ -3532,11 +3735,10 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo if (!first && blocks.empty()) { m_node_rpc_proxy.set_height(m_blockchain.size()); - refreshed = true; break; } if (!last) - tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, parsed_blocks, next_blocks, next_parsed_blocks, last, error, exception);}); + tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(first, try_incremental, start_height, next_blocks_start_height, short_chain_history, blocks, parsed_blocks, next_blocks, next_parsed_blocks, last, error, exception);}); if (!first) { @@ -3590,7 +3792,6 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo if(!first && blocks_start_height == next_blocks_start_height) { m_node_rpc_proxy.set_height(m_blockchain.size()); - refreshed = true; break; } @@ -3656,8 +3857,8 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo try { // If stop() is called we don't need to check pending transactions - if (check_pool && m_run.load(std::memory_order_relaxed) && !process_pool_txs.empty()) - process_pool_state(process_pool_txs); + if (check_pool && m_run.load(std::memory_order_relaxed) && !m_process_pool_txs.empty()) + process_pool_state(m_process_pool_txs); } catch (...) { @@ -3830,6 +4031,7 @@ bool wallet2::clear() m_subaddress_labels.clear(); m_multisig_rounds_passed = 0; m_device_last_key_image_sync = 0; + m_pool_info_query_time = 0; return true; } //---------------------------------------------------------------------------------------------------- @@ -3846,6 +4048,7 @@ void wallet2::clear_soft(bool keep_key_images) m_unconfirmed_payments.clear(); m_scanned_pool_txs[0].clear(); m_scanned_pool_txs[1].clear(); + m_pool_info_query_time = 0; cryptonote::block b; generate_genesis(b); @@ -8412,7 +8615,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> else { // the base offset of the first rct output in the first unlocked block (or the one to be if there's none) - num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; + num_outs = gamma->get_num_rct_outs(); LOG_PRINT_L1("" << num_outs << " unlocked rct outputs"); THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, "histogram reports no unlocked rct outputs, not even ours"); @@ -8696,7 +8899,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } bool use_histogram = amount != 0; if (!use_histogram) - num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; + num_outs = gamma->get_num_rct_outs(); // make sure the real outputs we asked for are really included, along // with the correct key and mask: this guards against an active attack @@ -9849,7 +10052,7 @@ void wallet2::light_wallet_get_address_txs() } } // TODO: purge old unconfirmed_txs - remove_obsolete_pool_txs(pool_txs); + remove_obsolete_pool_txs(pool_txs, false); // Calculate wallet balance m_light_wallet_balance = ires.total_received-wallet_total_sent; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 3ee40a5f0..40fa0bbad 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -101,6 +101,7 @@ namespace tools uint64_t pick(); gamma_picker(const std::vector<uint64_t> &rct_offsets); gamma_picker(const std::vector<uint64_t> &rct_offsets, double shape, double scale); + uint64_t get_num_rct_outs() const { return num_rct_outputs; } private: struct gamma_engine @@ -475,7 +476,7 @@ private: time_t m_sent_time; std::vector<cryptonote::tx_destination_entry> m_dests; crypto::hash m_payment_id; - enum { pending, pending_not_in_pool, failed } m_state; + enum { pending, pending_in_pool, failed } m_state; uint64_t m_timestamp; uint32_t m_subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> m_subaddr_indices; // set of address indices used as inputs in this transfer @@ -1023,7 +1024,7 @@ private: bool is_deprecated() const; void refresh(bool trusted_daemon); void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched); - void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true, uint64_t max_blocks = std::numeric_limits<uint64_t>::max()); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true, bool try_incremental = true, uint64_t max_blocks = std::numeric_limits<uint64_t>::max()); bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok); void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; } @@ -1506,9 +1507,9 @@ private: bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; - void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false); + void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false, bool try_incremental = false); void process_pool_state(const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &txs); - void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes); + void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes, bool remove_if_found); std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const; @@ -1706,11 +1707,16 @@ private: void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; bool clear(); void clear_soft(bool keep_key_images=false); - void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t ¤t_height); + void pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t ¤t_height); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); - void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception); + void pull_and_parse_next_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception); void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); + bool accept_pool_tx_for_processing(const crypto::hash &txid); + void process_unconfirmed_transfer(bool incremental, const crypto::hash &txid, wallet2::unconfirmed_transfer_details &tx_details, bool seen_in_pool, std::chrono::system_clock::time_point now, bool refreshed); + void process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed); + void update_pool_state_by_pool_query(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false); + void update_pool_state_from_pool_data(bool incremental, const std::vector<crypto::hash> &removed_pool_txids, const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &added_pool_txs, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed); uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); @@ -1845,6 +1851,8 @@ private: // If m_refresh_from_block_height is explicitly set to zero we need this to differentiate it from the case that // m_refresh_from_block_height was defaulted to zero.*/ bool m_explicit_refresh_from_block_height; + uint64_t m_pool_info_query_time; + std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> m_process_pool_txs; bool m_confirm_non_default_ring_size; AskPasswordType m_ask_password; uint64_t m_max_reorg_depth; diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 8474fb568..06910ebbb 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -155,8 +155,7 @@ bool wallet2::search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads const uint8_t major_version = hashing_blob[0]; if (major_version >= RX_BLOCK_VERSION) { - const int miners = 1; - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data, miners, 0); + crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data); } else { diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index cecf24289..768bc420a 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -154,7 +154,7 @@ namespace tools uint64_t blocks_fetched = 0; try { bool received_money = false; - if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, blocks_fetched, received_money, true, REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE); + if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, blocks_fetched, received_money, true, true, REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } |