diff options
Diffstat (limited to '')
26 files changed, 541 insertions, 354 deletions
diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 4a91cf7cc..a78d7ada9 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -52,9 +52,11 @@ using namespace cryptonote; static std::string db_path; // default to fast:1 -static uint64_t records_per_sync = 128; +static uint64_t records_per_sync = 16 * 65536; static const size_t slack = 512 * 1024 * 1024; +static std::vector<bool> is_v1; + static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) { std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); @@ -89,6 +91,14 @@ static void close(MDB_env *env) mdb_env_close(env); } +static void mark_v1_tx(const MDB_val &k, const MDB_val &v) +{ + const uint64_t tx_id = *(const uint64_t*)k.mv_data; + if (tx_id >= is_v1.size()) + is_v1.resize(tx_id + 1, false); + is_v1[tx_id] = cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + static void add_size(MDB_env *env, uint64_t bytes) { try @@ -136,7 +146,7 @@ static void check_resize(MDB_env *env, size_t bytes) add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); } -static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +static bool resize_point(size_t &nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) { if (nrecords % records_per_sync && bytes <= slack / 2) return false; @@ -146,10 +156,11 @@ static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &b dbr = mdb_txn_begin(env, NULL, 0, txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); bytes = 0; + nrecords = 0; return true; } -static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0, void (*f)(const MDB_val&, const MDB_val&) = 0) { MDB_dbi dbi0, dbi1; MDB_txn *txn0, *txn1; @@ -200,6 +211,11 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned dbr = mdb_cursor_open(txn1, dbi1, &cur1); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + if (flags & MDB_DUPSORT) + putflags |= MDB_APPENDDUP; + else + putflags |= MDB_APPEND; + MDB_val k; MDB_val v; MDB_cursor_op op = MDB_FIRST; @@ -214,7 +230,8 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); bytes += k.mv_size + v.mv_size; - if (resize_point(++nrecords, env1, &txn1, bytes)) + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) { dbr = mdb_cursor_open(txn1, dbi1, &cur1); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); @@ -223,6 +240,9 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned ret = mdb_cursor_put(cur1, &k, &v, putflags); if (ret) throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + + if (f) + (*f)(k, v); } mdb_cursor_close(cur1); @@ -235,17 +255,6 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned mdb_dbi_close(env0, dbi0); } -static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) -{ - MDB_val v; - int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); - if (ret) - throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); - if (v.mv_size == 0) - throw std::runtime_error("Invalid transaction pruned data"); - return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); -} - static void prune(MDB_env *env0, MDB_env *env1) { MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; @@ -324,7 +333,10 @@ static void prune(MDB_env *env0, MDB_env *env1) mdb_dbi_close(env0, dbi0_blocks); const uint64_t blockchain_height = stats.ms_entries; size_t nrecords = 0, bytes = 0; + std::vector<bool> prunable_needed; + // go through all txes tx indices, recording which ones should have their prunable part retained + MINFO("Marking prunable txes"); MDB_cursor_op op = MDB_FIRST; while (1) { @@ -336,7 +348,8 @@ static void prune(MDB_env *env0, MDB_env *env1) const txindex *ti = (const txindex*)v.mv_data; const uint64_t block_height = ti->data.block_id; - MDB_val_set(kk, ti->data.tx_id); + const uint64_t tx_id = ti->data.tx_id; + MDB_val_set(kk, tx_id); if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) { MDEBUG(block_height << "/" << blockchain_height << " is in tip"); @@ -344,22 +357,23 @@ static void prune(MDB_env *env0, MDB_env *env1) dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); bytes += kk.mv_size + vv.mv_size; - } - if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) - { - MDB_val vv; - dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); - if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); - bytes += kk.mv_size + vv.mv_size; - if (resize_point(++nrecords, env1, &txn1, bytes)) + + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) { dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); } - dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); - if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + if (tx_id >= is_v1.size()) + throw std::runtime_error("tx_id out of range of is_v1 vector"); + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1[tx_id]) + { + if (tx_id >= prunable_needed.size()) + prunable_needed.resize(tx_id + 1, false); + prunable_needed[tx_id] = true; } else { @@ -367,6 +381,37 @@ static void prune(MDB_env *env0, MDB_env *env1) } } + // go through prunable parts, carrying over those we need + MINFO("Copying retained prunable data"); + op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_txs_prunable, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const uint64_t tx_id = *(const uint64_t*)k.mv_data; + if (tx_id >= prunable_needed.size()) + throw std::runtime_error("tx_id out of range of prunable_needed vector"); + if (prunable_needed[tx_id]) + { + dbr = mdb_cursor_put(cur1_txs_prunable, &k, &v, MDB_APPEND); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + + bytes += k.mv_size + v.mv_size; + ++nrecords; + if (resize_point(nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + } + } + mdb_cursor_close(cur1_txs_prunable_tip); mdb_cursor_close(cur1_txs_prunable); mdb_cursor_close(cur0_txs_prunable); @@ -419,7 +464,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - records_per_sync = 1000; // default to fastest:async:1000 + // default to fastest:async:N } else return false; @@ -455,7 +500,7 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." - , "fast:1000" + , "fast:" + std::to_string(records_per_sync) }; const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; @@ -601,26 +646,27 @@ int main(int argc, char* argv[]) MDB_env *env0 = NULL, *env1 = NULL; open(env0, paths[0], db_flags, true); open(env1, paths[1], db_flags, false); - copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, 0); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); //copy_table(env0, env1, "txs", MDB_INTEGERKEY); - copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, 0, NULL, &mark_v1_tx); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0); // not copied: prunable, prunable_tip copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); - copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); - copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, 0); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "alt_blocks", 0, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, 0); copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); if (already_pruned) { - copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); - copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, 0, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_uint64); } else { diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index e00421f87..466224390 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -30,6 +30,8 @@ // check local first (in the event of static or in-source compilation of libunbound) #include "unbound.h" +#include <deque> +#include <set> #include <stdlib.h> #include "include_base_utils.h" #include "common/threadpool.h" @@ -326,11 +328,6 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec dnssec_available = false; dnssec_valid = false; - if (!check_address_syntax(url.c_str())) - { - return addresses; - } - // destructor takes care of cleanup ub_result_ptr result; @@ -413,16 +410,6 @@ DNSResolver DNSResolver::create() return DNSResolver(); } -bool DNSResolver::check_address_syntax(const char *addr) const -{ - // if string doesn't contain a dot, we won't consider it a url for now. - if (strchr(addr,'.') == NULL) - { - return false; - } - return true; -} - namespace dns_utils { diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index f9507b42a..81079ba30 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -159,15 +159,6 @@ private: // TODO: modify this to accommodate DNSSEC std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); - /** - * @brief Checks a string to see if it looks like a URL - * - * @param addr the string to be checked - * - * @return true if it looks enough like a URL, false if not - */ - bool check_address_syntax(const char *addr) const; - DNSResolverData *m_data; }; // class DNSResolver diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 53421e18b..fcf8ca945 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -31,6 +31,7 @@ #include <boost/thread/mutex.hpp> #include <boost/thread/thread.hpp> #include <cstddef> +#include <deque> #include <functional> #include <utility> #include <vector> diff --git a/src/crypto/c_threads.h b/src/crypto/c_threads.h index c5431cb8d..b4f773641 100644 --- a/src/crypto/c_threads.h +++ b/src/crypto/c_threads.h @@ -30,29 +30,39 @@ #pragma once #ifdef _WIN32 + #include <windows.h> -#define CTHR_MUTEX_TYPE HANDLE -#define CTHR_MUTEX_INIT NULL -#define CTHR_MUTEX_LOCK(x) do { if (x == NULL) { \ - HANDLE p = CreateMutex(NULL, FALSE, NULL); \ - if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \ - CloseHandle(p); \ - } WaitForSingleObject(x, INFINITE); } while(0) -#define CTHR_MUTEX_UNLOCK(x) ReleaseMutex(x) + +#define CTHR_RWLOCK_TYPE SRWLOCK +#define CTHR_RWLOCK_INIT SRWLOCK_INIT +#define CTHR_RWLOCK_LOCK_WRITE(x) AcquireSRWLockExclusive(&x) +#define CTHR_RWLOCK_UNLOCK_WRITE(x) ReleaseSRWLockExclusive(&x) +#define CTHR_RWLOCK_LOCK_READ(x) AcquireSRWLockShared(&x) +#define CTHR_RWLOCK_UNLOCK_READ(x) ReleaseSRWLockShared(&x) +#define CTHR_RWLOCK_TRYLOCK_READ(x) TryAcquireSRWLockShared(&x) + #define CTHR_THREAD_TYPE HANDLE #define CTHR_THREAD_RTYPE void #define CTHR_THREAD_RETURN return -#define CTHR_THREAD_CREATE(thr, func, arg) thr = (HANDLE)_beginthread(func, 0, arg) -#define CTHR_THREAD_JOIN(thr) WaitForSingleObject(thr, INFINITE) +#define CTHR_THREAD_CREATE(thr, func, arg) ((thr = (HANDLE)_beginthread(func, 0, arg)) != -1L) +#define CTHR_THREAD_JOIN(thr) WaitForSingleObject((HANDLE)thr, INFINITE) + #else + #include <pthread.h> -#define CTHR_MUTEX_TYPE pthread_mutex_t -#define CTHR_MUTEX_INIT PTHREAD_MUTEX_INITIALIZER -#define CTHR_MUTEX_LOCK(x) pthread_mutex_lock(&x) -#define CTHR_MUTEX_UNLOCK(x) pthread_mutex_unlock(&x) + +#define CTHR_RWLOCK_TYPE pthread_rwlock_t +#define CTHR_RWLOCK_INIT PTHREAD_RWLOCK_INITIALIZER +#define CTHR_RWLOCK_LOCK_WRITE(x) pthread_rwlock_wrlock(&x) +#define CTHR_RWLOCK_UNLOCK_WRITE(x) pthread_rwlock_unlock(&x) +#define CTHR_RWLOCK_LOCK_READ(x) pthread_rwlock_rdlock(&x) +#define CTHR_RWLOCK_UNLOCK_READ(x) pthread_rwlock_unlock(&x) +#define CTHR_RWLOCK_TRYLOCK_READ(x) (pthread_rwlock_tryrdlock(&x) == 0) + #define CTHR_THREAD_TYPE pthread_t #define CTHR_THREAD_RTYPE void * #define CTHR_THREAD_RETURN return NULL -#define CTHR_THREAD_CREATE(thr, func, arg) pthread_create(&thr, NULL, func, arg) +#define CTHR_THREAD_CREATE(thr, func, arg) (pthread_create(&thr, NULL, func, arg) == 0) #define CTHR_THREAD_JOIN(thr) pthread_join(thr, NULL) + #endif diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 4b392d472..971bf663f 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -38,7 +38,6 @@ DISABLE_VS_WARNINGS(4146 4244) /* Predeclarations */ -static void fe_mul(fe, const fe, const fe); static void fe_sq(fe, const fe); static void ge_madd(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_msub(ge_p1p1 *, const ge_p3 *, const ge_precomp *); @@ -72,7 +71,7 @@ uint64_t load_4(const unsigned char *in) h = 0 */ -static void fe_0(fe h) { +void fe_0(fe h) { h[0] = 0; h[1] = 0; h[2] = 0; @@ -375,7 +374,7 @@ Can get away with 11 carries, but then data flow is much deeper. With tighter constraints on inputs can squeeze carries into int32. */ -static void fe_mul(fe h, const fe f, const fe g) { +void fe_mul(fe h, const fe f, const fe g) { int32_t f0 = f[0]; int32_t f1 = f[1]; int32_t f2 = f[2]; diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index e4901e080..7a6f0789a 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -30,6 +30,8 @@ #pragma once +#include <stdint.h> + /* From fe.h */ typedef int32_t fe[10]; @@ -161,5 +163,7 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); +void fe_mul(fe out, const fe, const fe); +void fe_0(fe h); int ge_p3_is_point_at_infinity_vartime(const ge_p3 *p); diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index d8cd6c6a0..43ea59ac6 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -335,8 +335,16 @@ namespace crypto { inline bool operator<(const public_key &p1, const public_key &p2) { return memcmp(&p1, &p2, sizeof(public_key)) < 0; } inline bool operator>(const public_key &p1, const public_key &p2) { return p2 < p1; } + inline bool operator<(const key_image &p1, const key_image &p2) { return memcmp(&p1, &p2, sizeof(key_image)) < 0; } + inline bool operator>(const key_image &p1, const key_image &p2) { return p2 < p1; } } +// type conversions for easier calls to sc_add(), sc_sub(), hash functions +inline unsigned char* to_bytes(crypto::ec_scalar &scalar) { return &reinterpret_cast<unsigned char&>(scalar); } +inline const unsigned char* to_bytes(const crypto::ec_scalar &scalar) { return &reinterpret_cast<const unsigned char&>(scalar); } +inline unsigned char* to_bytes(crypto::ec_point &point) { return &reinterpret_cast<unsigned char&>(point); } +inline const unsigned char* to_bytes(const crypto::ec_point &point) { return &reinterpret_cast<const unsigned char&>(point); } + CRYPTO_MAKE_HASHABLE(public_key) CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(public_key_memsafe) diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index b7ec80d7c..9d3abc3f8 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -97,5 +97,9 @@ void rx_slow_hash_allocate_state(void); void rx_slow_hash_free_state(void); uint64_t rx_seedheight(const uint64_t height); void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height); -void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash, int miners, int is_alt); -void rx_reorg(const uint64_t split_height); + +void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads); +void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash); + +void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads); +uint32_t rx_get_miner_thread(void); diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index 40ef96ac9..8682daeb2 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -43,32 +43,38 @@ #define RX_LOGCAT "randomx" +static CTHR_RWLOCK_TYPE main_dataset_lock = CTHR_RWLOCK_INIT; +static CTHR_RWLOCK_TYPE main_cache_lock = CTHR_RWLOCK_INIT; + +static randomx_dataset *main_dataset = NULL; +static randomx_cache *main_cache = NULL; +static char main_seedhash[HASH_SIZE]; +static int main_seedhash_set = 0; + +static CTHR_RWLOCK_TYPE secondary_cache_lock = CTHR_RWLOCK_INIT; + +static randomx_cache *secondary_cache = NULL; +static char secondary_seedhash[HASH_SIZE]; +static int secondary_seedhash_set = 0; + #if defined(_MSC_VER) #define THREADV __declspec(thread) #else #define THREADV __thread #endif -typedef struct rx_state { - CTHR_MUTEX_TYPE rs_mutex; - char rs_hash[HASH_SIZE]; - uint64_t rs_height; - randomx_cache *rs_cache; -} rx_state; +static THREADV randomx_vm *main_vm_full = NULL; +static THREADV randomx_vm *main_vm_light = NULL; +static THREADV randomx_vm *secondary_vm_light = NULL; -static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT; -static CTHR_MUTEX_TYPE rx_dataset_mutex = CTHR_MUTEX_INIT; +static THREADV uint32_t miner_thread = 0; -static rx_state rx_s[2] = {{CTHR_MUTEX_INIT,{0},0,0},{CTHR_MUTEX_INIT,{0},0,0}}; - -static randomx_dataset *rx_dataset; -static int rx_dataset_nomem; -static int rx_dataset_nolp; -static uint64_t rx_dataset_height; -static THREADV randomx_vm *rx_vm = NULL; +static bool is_main(const char* seedhash) { return main_seedhash_set && (memcmp(seedhash, main_seedhash, HASH_SIZE) == 0); } +static bool is_secondary(const char* seedhash) { return secondary_seedhash_set && (memcmp(seedhash, secondary_seedhash, HASH_SIZE) == 0); } static void local_abort(const char *msg) { + merror(RX_LOGCAT, "%s", msg); fprintf(stderr, "%s\n", msg); #ifdef NDEBUG _exit(1); @@ -77,6 +83,16 @@ static void local_abort(const char *msg) #endif } +static void hash2hex(const char* hash, char* hex) { + const char* d = "0123456789abcdef"; + for (int i = 0; i < HASH_SIZE; ++i) { + const uint8_t b = hash[i]; + hex[i * 2 + 0] = d[b >> 4]; + hex[i * 2 + 1] = d[b & 15]; + } + hex[HASH_SIZE * 2] = '\0'; +} + static inline int disabled_flags(void) { static int flags = -1; @@ -157,19 +173,6 @@ static unsigned int get_seedhash_epoch_blocks(void) return blocks; } -void rx_reorg(const uint64_t split_height) { - int i; - CTHR_MUTEX_LOCK(rx_mutex); - for (i=0; i<2; i++) { - if (split_height <= rx_s[i].rs_height) { - if (rx_s[i].rs_height == rx_dataset_height) - rx_dataset_height = 1; - rx_s[i].rs_height = 1; /* set to an invalid seed height */ - } - } - CTHR_MUTEX_UNLOCK(rx_mutex); -} - uint64_t rx_seedheight(const uint64_t height) { const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag(); const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks(); @@ -183,6 +186,95 @@ void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nexth *nextheight = rx_seedheight(height + get_seedhash_epoch_lag()); } +static void rx_alloc_dataset(randomx_flags flags, randomx_dataset** dataset, int ignore_env) +{ + if (*dataset) { + return; + } + + if (disabled_flags() & RANDOMX_FLAG_FULL_MEM) { + static int shown = 0; + if (!shown) { + shown = 1; + minfo(RX_LOGCAT, "RandomX dataset is disabled by MONERO_RANDOMX_UMASK environment variable."); + } + return; + } + + if (!ignore_env && !getenv("MONERO_RANDOMX_FULL_MEM")) { + static int shown = 0; + if (!shown) { + shown = 1; + minfo(RX_LOGCAT, "RandomX dataset is not enabled by default. Use MONERO_RANDOMX_FULL_MEM environment variable to enable it."); + } + return; + } + + *dataset = randomx_alloc_dataset((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags()); + if (!*dataset) { + mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset using large pages"); + *dataset = randomx_alloc_dataset(flags & ~disabled_flags()); + if (!*dataset) { + merror(RX_LOGCAT, "Couldn't allocate RandomX dataset"); + } + } +} + +static void rx_alloc_cache(randomx_flags flags, randomx_cache** cache) +{ + if (*cache) { + return; + } + + *cache = randomx_alloc_cache((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags()); + if (!*cache) { + mwarning(RX_LOGCAT, "Couldn't allocate RandomX cache using large pages"); + *cache = randomx_alloc_cache(flags & ~disabled_flags()); + if (!*cache) local_abort("Couldn't allocate RandomX cache"); + } +} + +static void rx_init_full_vm(randomx_flags flags, randomx_vm** vm) +{ + if (*vm || !main_dataset || (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) { + return; + } + + if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) { + flags |= RANDOMX_FLAG_SECURE; + } + + *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset); + if (!*vm) { + mwarning(RX_LOGCAT, "Couldn't allocate RandomX full VM using large pages"); + *vm = randomx_create_vm((flags | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset); + if (!*vm) { + merror(RX_LOGCAT, "Couldn't allocate RandomX full VM"); + } + } +} + +static void rx_init_light_vm(randomx_flags flags, randomx_vm** vm, randomx_cache* cache) +{ + if (*vm) { + randomx_vm_set_cache(*vm, cache); + return; + } + + if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) { + flags |= RANDOMX_FLAG_SECURE; + } + + flags &= ~RANDOMX_FLAG_FULL_MEM; + + *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags(), cache, NULL); + if (!*vm) { + mwarning(RX_LOGCAT, "Couldn't allocate RandomX light VM using large pages"); + *vm = randomx_create_vm(flags & ~disabled_flags(), cache, NULL); + if (!*vm) local_abort("Couldn't allocate RandomX light VM"); + } +} + typedef struct seedinfo { randomx_cache *si_cache; unsigned long si_start; @@ -191,187 +283,230 @@ typedef struct seedinfo { static CTHR_THREAD_RTYPE rx_seedthread(void *arg) { seedinfo *si = arg; - randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count); + randomx_init_dataset(main_dataset, si->si_cache, si->si_start, si->si_count); CTHR_THREAD_RETURN; } -static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_t seedheight) { - if (miners > 1) { - unsigned long delta = randomx_dataset_item_count() / miners; - unsigned long start = 0; - int i; - seedinfo *si; - CTHR_THREAD_TYPE *st; - si = malloc(miners * sizeof(seedinfo)); - if (si == NULL) - local_abort("Couldn't allocate RandomX mining threadinfo"); - st = malloc(miners * sizeof(CTHR_THREAD_TYPE)); - if (st == NULL) { - free(si); - local_abort("Couldn't allocate RandomX mining threadlist"); - } - for (i=0; i<miners-1; i++) { - si[i].si_cache = rs_cache; - si[i].si_start = start; - si[i].si_count = delta; - start += delta; - } - si[i].si_cache = rs_cache; +static void rx_init_dataset(size_t max_threads) { + if (!main_dataset) { + return; + } + + // leave 2 CPU cores for other tasks + const size_t num_threads = (max_threads < 4) ? 1 : (max_threads - 2); + seedinfo* si = malloc(num_threads * sizeof(seedinfo)); + if (!si) local_abort("Couldn't allocate RandomX mining threadinfo"); + + const uint32_t delta = randomx_dataset_item_count() / num_threads; + uint32_t start = 0; + + const size_t n1 = num_threads - 1; + for (size_t i = 0; i < n1; ++i) { + si[i].si_cache = main_cache; si[i].si_start = start; - si[i].si_count = randomx_dataset_item_count() - start; - for (i=1; i<miners; i++) { - CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i]); - } - randomx_init_dataset(rx_dataset, rs_cache, 0, si[0].si_count); - for (i=1; i<miners; i++) { - CTHR_THREAD_JOIN(st[i]); + si[i].si_count = delta; + start += delta; + } + + si[n1].si_cache = main_cache; + si[n1].si_start = start; + si[n1].si_count = randomx_dataset_item_count() - start; + + CTHR_THREAD_TYPE *st = malloc(num_threads * sizeof(CTHR_THREAD_TYPE)); + if (!st) local_abort("Couldn't allocate RandomX mining threadlist"); + + CTHR_RWLOCK_LOCK_READ(main_cache_lock); + for (size_t i = 0; i < n1; ++i) { + if (!CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i])) { + local_abort("Couldn't start RandomX seed thread"); } - free(st); - free(si); - } else { - randomx_init_dataset(rx_dataset, rs_cache, 0, randomx_dataset_item_count()); } - rx_dataset_height = seedheight; + rx_seedthread(&si[n1]); + for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]); + CTHR_RWLOCK_UNLOCK_READ(main_cache_lock); + + free(st); + free(si); + + minfo(RX_LOGCAT, "RandomX dataset initialized"); } -void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, - char *hash, int miners, int is_alt) { - uint64_t s_height = rx_seedheight(mainheight); - int toggle = (s_height & get_seedhash_epoch_blocks()) != 0; - randomx_flags flags = enabled_flags() & ~disabled_flags(); - rx_state *rx_sp; - randomx_cache *cache; - - CTHR_MUTEX_LOCK(rx_mutex); - - /* if alt block but with same seed as mainchain, no need for alt cache */ - if (is_alt) { - if (s_height == seedheight && !memcmp(rx_s[toggle].rs_hash, seedhash, HASH_SIZE)) - is_alt = 0; - } else { - /* RPC could request an earlier block on mainchain */ - if (s_height > seedheight) - is_alt = 1; - /* miner can be ahead of mainchain */ - else if (s_height < seedheight) - toggle ^= 1; - } - - toggle ^= (is_alt != 0); - - rx_sp = &rx_s[toggle]; - CTHR_MUTEX_LOCK(rx_sp->rs_mutex); - CTHR_MUTEX_UNLOCK(rx_mutex); - - cache = rx_sp->rs_cache; - if (cache == NULL) { - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { - cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); - if (cache == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache"); - } - } - if (cache == NULL) { - cache = randomx_alloc_cache(flags); - if (cache == NULL) - local_abort("Couldn't allocate RandomX cache"); - } +typedef struct thread_info { + char seedhash[HASH_SIZE]; + size_t max_threads; +} thread_info; + +static CTHR_THREAD_RTYPE rx_set_main_seedhash_thread(void *arg) { + thread_info* info = arg; + + CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock); + CTHR_RWLOCK_LOCK_WRITE(main_cache_lock); + + // Double check that seedhash wasn't already updated + if (is_main(info->seedhash)) { + CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock); + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + free(info); + CTHR_THREAD_RETURN; } - if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) { - randomx_init_cache(cache, seedhash, HASH_SIZE); - rx_sp->rs_cache = cache; - rx_sp->rs_height = seedheight; - memcpy(rx_sp->rs_hash, seedhash, HASH_SIZE); + memcpy(main_seedhash, info->seedhash, HASH_SIZE); + main_seedhash_set = 1; + + char buf[HASH_SIZE * 2 + 1]; + hash2hex(main_seedhash, buf); + minfo(RX_LOGCAT, "RandomX new main seed hash is %s", buf); + + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + rx_alloc_dataset(flags, &main_dataset, 0); + rx_alloc_cache(flags, &main_cache); + + randomx_init_cache(main_cache, info->seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX main cache initialized"); + + CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock); + + // From this point, rx_slow_hash can calculate hashes in light mode, but dataset is not initialized yet + rx_init_dataset(info->max_threads); + + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + + free(info); + CTHR_THREAD_RETURN; +} + +void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads) { + // Early out if seedhash didn't change + if (is_main(seedhash)) { + return; } - if (rx_vm == NULL) { - if ((flags & RANDOMX_FLAG_JIT) && !miners) { - flags |= RANDOMX_FLAG_SECURE & ~disabled_flags(); - } - if (miners && (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) { - miners = 0; - } - if (miners) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (!rx_dataset_nomem) { - if (rx_dataset == NULL) { - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); - if (rx_dataset == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); - } - } - if (rx_dataset == NULL) - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); - if (rx_dataset != NULL) - rx_initdata(rx_sp->rs_cache, miners, seedheight); - } - } - if (rx_dataset != NULL) - flags |= RANDOMX_FLAG_FULL_MEM; - else { - miners = 0; - if (!rx_dataset_nomem) { - rx_dataset_nomem = 1; - mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner"); + + // Update main cache and dataset in the background + thread_info* info = malloc(sizeof(thread_info)); + if (!info) local_abort("Couldn't allocate RandomX mining threadinfo"); + + memcpy(info->seedhash, seedhash, HASH_SIZE); + info->max_threads = max_dataset_init_threads; + + CTHR_THREAD_TYPE t; + if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) { + local_abort("Couldn't start RandomX seed thread"); + } +} + +void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) { + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + int success = 0; + + // Fast path (seedhash == main_seedhash) + // Multiple threads can run in parallel in fast or light mode, 1-2 ms or 10-15 ms per hash per thread + if (is_main(seedhash)) { + // If CTHR_RWLOCK_TRYLOCK_READ fails it means dataset is being initialized now, so use the light mode + if (main_dataset && CTHR_RWLOCK_TRYLOCK_READ(main_dataset_lock)) { + // Double check that main_seedhash didn't change + if (is_main(seedhash)) { + rx_init_full_vm(flags, &main_vm_full); + if (main_vm_full) { + randomx_calculate_hash(main_vm_full, data, length, result_hash); + success = 1; } } - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); - } - if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES) && !rx_dataset_nolp) { - rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) { //large pages failed - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); - rx_dataset_nolp = 1; + CTHR_RWLOCK_UNLOCK_READ(main_dataset_lock); + } else { + CTHR_RWLOCK_LOCK_READ(main_cache_lock); + // Double check that main_seedhash didn't change + if (is_main(seedhash)) { + rx_init_light_vm(flags, &main_vm_light, main_cache); + randomx_calculate_hash(main_vm_light, data, length, result_hash); + success = 1; } + CTHR_RWLOCK_UNLOCK_READ(main_cache_lock); } - if (rx_vm == NULL) - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) {//fallback if everything fails - flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); - } - if (rx_vm == NULL) - local_abort("Couldn't allocate RandomX VM"); - } else if (miners) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (rx_dataset != NULL && rx_dataset_height != seedheight) - rx_initdata(cache, miners, seedheight); - else if (rx_dataset == NULL) { - /* this is a no-op if the cache hasn't changed */ - randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); + } + + if (success) { + return; + } + + char buf[HASH_SIZE * 2 + 1]; + + // Slow path (seedhash != main_seedhash, but seedhash == secondary_seedhash) + // Multiple threads can run in parallel in light mode, 10-15 ms per hash per thread + if (!secondary_cache) { + CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock); + if (!secondary_cache) { + hash2hex(seedhash, buf); + minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf); + + rx_alloc_cache(flags, &secondary_cache); + randomx_init_cache(secondary_cache, seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX secondary cache updated"); + memcpy(secondary_seedhash, seedhash, HASH_SIZE); + secondary_seedhash_set = 1; } - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); - } else { - /* this is a no-op if the cache hasn't changed */ - randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); - } - /* mainchain users can run in parallel */ - if (!is_alt) - CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex); - randomx_calculate_hash(rx_vm, data, length, hash); - /* altchain slot users always get fully serialized */ - if (is_alt) - CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex); -} + CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock); + } -void rx_slow_hash_allocate_state(void) { + CTHR_RWLOCK_LOCK_READ(secondary_cache_lock); + if (is_secondary(seedhash)) { + rx_init_light_vm(flags, &secondary_vm_light, secondary_cache); + randomx_calculate_hash(secondary_vm_light, data, length, result_hash); + success = 1; + } + CTHR_RWLOCK_UNLOCK_READ(secondary_cache_lock); + + if (success) { + return; + } + + // Slowest path (seedhash != main_seedhash, seedhash != secondary_seedhash) + // Only one thread runs at a time and updates secondary_seedhash if needed, up to 200-500 ms per hash + CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock); + if (!is_secondary(seedhash)) { + hash2hex(seedhash, buf); + minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf); + + randomx_init_cache(secondary_cache, seedhash, HASH_SIZE); + minfo(RX_LOGCAT, "RandomX secondary cache updated"); + memcpy(secondary_seedhash, seedhash, HASH_SIZE); + secondary_seedhash_set = 1; + } + rx_init_light_vm(flags, &secondary_vm_light, secondary_cache); + randomx_calculate_hash(secondary_vm_light, data, length, result_hash); + CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock); } -void rx_slow_hash_free_state(void) { - if (rx_vm != NULL) { - randomx_destroy_vm(rx_vm); - rx_vm = NULL; +void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads) { + miner_thread = value; + + // If dataset is not allocated yet, try to allocate and initialize it + CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock); + if (main_dataset) { + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); + return; } + + const randomx_flags flags = enabled_flags() & ~disabled_flags(); + rx_alloc_dataset(flags, &main_dataset, 1); + rx_init_dataset(max_dataset_init_threads); + + CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock); +} + +uint32_t rx_get_miner_thread() { + return miner_thread; } -void rx_stop_mining(void) { - CTHR_MUTEX_LOCK(rx_dataset_mutex); - if (rx_dataset != NULL) { - randomx_dataset *rd = rx_dataset; - rx_dataset = NULL; - randomx_release_dataset(rd); +void rx_slow_hash_allocate_state() {} + +static void rx_destroy_vm(randomx_vm** vm) { + if (*vm) { + randomx_destroy_vm(*vm); + *vm = NULL; } - rx_dataset_nomem = 0; - rx_dataset_nolp = 0; - CTHR_MUTEX_UNLOCK(rx_dataset_mutex); +} + +void rx_slow_hash_free_state() { + rx_destroy_vm(&main_vm_full); + rx_destroy_vm(&main_vm_light); + rx_destroy_vm(&secondary_vm_light); } diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 5b0db9518..98f1555b6 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -82,6 +82,7 @@ using namespace epee; #include "miner.h" +#include "crypto/hash.h" extern "C" void slow_hash_allocate_state(); @@ -436,7 +437,6 @@ namespace cryptonote { m_stop = true; } - extern "C" void rx_stop_mining(void); //----------------------------------------------------------------------------------------------------- bool miner::stop() { @@ -469,7 +469,6 @@ namespace cryptonote MINFO("Mining has been stopped, " << m_threads.size() << " finished" ); m_threads.clear(); m_threads_autodetect.clear(); - rx_stop_mining(); return true; } //----------------------------------------------------------------------------------------------------- @@ -524,6 +523,8 @@ namespace cryptonote bool miner::worker_thread() { const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment + crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency()); + MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]"); MGINFO("Miner thread was started ["<< th_local_index << "]"); uint32_t nonce = m_starter_nonce + th_local_index; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 3b7234a05..b1903cf6c 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -455,6 +455,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline if (!update_next_cumulative_weight_limit()) return false; } + + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height())); + if (seedhash != crypto::null_hash) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + } + return true; } //------------------------------------------------------------------ @@ -569,6 +577,12 @@ void Blockchain::pop_blocks(uint64_t nblocks) if (stop_batch) m_db->batch_stop(); + + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height())); + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + } } //------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the @@ -1238,18 +1252,20 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> } m_hardfork->reorganize_from_chain_height(split_height); - get_block_longhash_reorg(split_height); std::shared_ptr<tools::Notify> reorg_notify = m_reorg_notify; if (reorg_notify) reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + const uint64_t new_height = m_db->height(); + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height)); + crypto::hash prev_id; if (!get_block_hash(alt_chain.back().bl, prev_id)) MERROR("Failed to get block hash of an alternative chain's tip"); else - send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + send_miner_notifications(new_height, seedhash, prev_id, alt_chain.back().already_generated_coins); for (const auto& notifier : m_block_notifiers) { @@ -1261,6 +1277,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> } } + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height()); return true; } @@ -2000,7 +2019,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { seedhash = get_block_id_by_height(seedheight); } - get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash); + get_altblock_longhash(bei.bl, proof_of_work, seedhash); } else { get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0); @@ -4551,11 +4570,15 @@ leave: } } - send_miner_notifications(id, already_generated_coins); + const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height)); + send_miner_notifications(new_height, seedhash, id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency()); + return true; } //------------------------------------------------------------------ @@ -5760,24 +5783,15 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_valid = true; } -void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins) +void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins) { if (m_miner_notifiers.empty()) return; - const uint64_t height = m_db->height(); const uint8_t major_version = m_hardfork->get_ideal_version(height); const difficulty_type diff = get_difficulty_for_next_block(); const uint64_t median_weight = m_current_block_cumul_weight_median; - crypto::hash seed_hash = crypto::null_hash; - if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) - { - uint64_t seed_height, next_height; - crypto::rx_seedheights(height, &seed_height, &next_height); - seed_hash = get_block_id_by_height(seed_height); - } - std::vector<tx_block_template_backlog_entry> tx_backlog; m_tx_pool.get_block_template_backlog(tx_backlog); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 4795fc55c..c61ce4466 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -1598,9 +1598,11 @@ namespace cryptonote /** * @brief sends new block notifications to ZMQ `miner_data` subscribers * + * @param height current blockchain height + * @param seed_hash seed hash to use for mining * @param prev_id hash of new blockchain tip * @param already_generated_coins total coins mined by the network so far */ - void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); + void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 472026217..bf58a120d 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -669,10 +669,10 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash) + void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash) { blobdata bd = get_block_hashing_blob(b); - rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1); + rx_slow_hash(seed_hash.data, bd.data(), bd.size(), res.data); } bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners) @@ -686,20 +686,16 @@ namespace cryptonote } if (major_version >= RX_BLOCK_VERSION) { - uint64_t seed_height, main_height; crypto::hash hash; if (pbc != NULL) { - seed_height = rx_seedheight(height); + const uint64_t seed_height = rx_seedheight(height); hash = seed_hash ? *seed_hash : pbc->get_pending_block_id_by_height(seed_height); - main_height = pbc->get_current_blockchain_height(); } else { memset(&hash, 0, sizeof(hash)); // only happens when generating genesis block - seed_height = 0; - main_height = 0; } - rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash); + rx_slow_hash(hash.data, bd.data(), bd.size(), res.data); } else { const int pow_variant = major_version >= 7 ? major_version - 6 : 0; crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height); @@ -713,20 +709,10 @@ namespace cryptonote return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners); } - bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners) - { - return get_block_longhash(pbc, b, res, height, NULL, miners); - } - - crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners) + crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const crypto::hash *seed_hash, const int miners) { crypto::hash p = crypto::null_hash; - get_block_longhash(pbc, b, p, height, miners); + get_block_longhash(pbc, b, p, height, seed_hash, miners); return p; } - - void get_block_longhash_reorg(const uint64_t split_height) - { - rx_reorg(split_height); - } } diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 12d6b8ce5..5f301bb89 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -144,14 +144,10 @@ namespace cryptonote ); class Blockchain; - bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, - const int major_version, const crypto::hash *seed_hash, const int miners); - bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners); - bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners); - void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, - const uint64_t seed_height, const crypto::hash& seed_hash); - crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners); - void get_block_longhash_reorg(const uint64_t split_height); + bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners = 0); + bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0); + crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0); + void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash); } diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 83f37015f..27c6d0278 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -542,6 +542,7 @@ namespace levin i_core_events* core_; std::vector<blobdata> txs_; boost::uuids::uuid source_; + relay_method tx_relay; //! \pre Called in `zone_->strand` void operator()() @@ -549,7 +550,7 @@ namespace levin if (!zone_ || !core_ || txs_.empty()) return; - if (!zone_->fluffing) + if (!zone_->fluffing || tx_relay == relay_method::local) { core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem); for (int tries = 2; 0 < tries; tries--) @@ -589,7 +590,7 @@ namespace levin change_channels(change_channels&&) = default; change_channels(const change_channels& source) - : zone_(source.zone_), map_(source.map_.clone()) + : zone_(source.zone_), map_(source.map_.clone()), fluffing_(source.fluffing_) {} //! \pre Called within `zone_->strand`. @@ -871,7 +872,7 @@ namespace levin { // this will change a local/forward tx to stem or fluff ... zone_->strand.dispatch( - dandelionpp_notify{zone_, core_, std::move(txs), source} + dandelionpp_notify{zone_, core_, std::move(txs), source, tx_relay} ); break; } diff --git a/src/net/parse.cpp b/src/net/parse.cpp index 1df6175b4..92be492a3 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -38,7 +38,7 @@ namespace net { void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port) { - // require ipv6 address format "[addr:addr:addr:...:addr]:port" + // If IPv6 address format with port "[addr:addr:addr:...:addr]:port" if (address.find(']') != std::string::npos) { host = address.substr(1, address.rfind(']') - 1); @@ -47,6 +47,12 @@ namespace net port = address.substr(address.rfind(':') + 1); } } + // Else if IPv6 address format without port e.g. "addr:addr:addr:...:addr" + else if (std::count(address.begin(), address.end(), ':') >= 2) + { + host = address; + } + // Else IPv4, Tor, I2P address or hostname else { host = address.substr(0, address.rfind(':')); diff --git a/src/net/parse.h b/src/net/parse.h index 648076d7b..6ece931c6 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -38,6 +38,16 @@ namespace net { + /*! + * \brief Takes a valid address string (IP, Tor, I2P, or DNS name) and splits it into host and port + * + * The host of an IPv6 addresses in the format "[x:x:..:x]:port" will have the braces stripped. + * For example, when the address is "[ffff::2023]", host will be set to "ffff::2023". + * + * \param address The address string one wants to split + * \param[out] host The host part of the address string. Is always set. + * \param[out] port The port part of the address string. Is only set when address string contains a port. + */ void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port); /*! diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index f33ce977d..df67734d5 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -645,20 +645,10 @@ namespace nodetool { using namespace boost::asio; - std::string host = addr; + // Split addr string into host string and port string + std::string host; std::string port = std::to_string(default_port); - size_t colon_pos = addr.find_last_of(':'); - size_t dot_pos = addr.find_last_of('.'); - size_t square_brace_pos = addr.find('['); - - // IPv6 will have colons regardless. IPv6 and IPv4 address:port will have a colon but also either a . or a [ - // as IPv6 addresses specified as address:port are to be specified as "[addr:addr:...:addr]:port" - // One may also specify an IPv6 address as simply "[addr:addr:...:addr]" without the port; in that case - // the square braces will be stripped here. - if ((std::string::npos != colon_pos && std::string::npos != dot_pos) || std::string::npos != square_brace_pos) - { - net::get_network_address_host_and_port(addr, host, port); - } + net::get_network_address_host_and_port(addr, host, port); MINFO("Resolving node address: host=" << host << ", port=" << port); io_service io_srv; @@ -2413,7 +2403,7 @@ namespace nodetool return false; } return true; - }); + }, "0.0.0.0", m_ssl_support); if(!r) { LOG_WARNING_CC(context, "Failed to call connect_async, network error."); diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 245a3f477..5b039d907 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -671,7 +671,7 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - static key ecdhHash(const key &k) + key genAmountEncodingFactor(const key &k) { char data[38]; rct::key hash; @@ -700,7 +700,7 @@ namespace rct { if (v2) { unmasked.mask = zero(); - xor8(unmasked.amount, ecdhHash(sharedSec)); + xor8(unmasked.amount, genAmountEncodingFactor(sharedSec)); } else { @@ -715,7 +715,7 @@ namespace rct { if (v2) { masked.mask = genCommitmentMask(sharedSec); - xor8(masked.amount, ecdhHash(sharedSec)); + xor8(masked.amount, genAmountEncodingFactor(sharedSec)); } else { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 679ed1441..da0013ee1 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -184,6 +184,7 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH + key genAmountEncodingFactor(const key &k); key genCommitmentMask(const key &sk); void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 2e05f04fb..aca3e3761 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -236,10 +236,8 @@ namespace cryptonote *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(nonce); if (block.major_version >= RX_BLOCK_VERSION) { - const uint64_t seed_height = is_current ? info.seed_height : info.previous_seed_height; const crypto::hash &seed_hash = is_current ? info.seed_hash : info.previous_seed_hash; - const uint64_t height = cryptonote::get_block_height(block); - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, 0, 0); + crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data); } else { diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index c4d3856d4..085f4f9df 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -535,7 +535,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas view_wallet->generate(path, password, address, viewkey); // Export/Import outputs - auto outputs = m_wallet->export_outputs(); + auto outputs = m_wallet->export_outputs(true/*all*/); view_wallet->import_outputs(outputs); // Copy scanned blockchain @@ -553,7 +553,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas // Export/Import key images // We already know the spent status from the outputs we exported, thus no need to check them again - auto key_images = m_wallet->export_key_images(); + auto key_images = m_wallet->export_key_images(true/*all*/); uint64_t spent = 0; uint64_t unspent = 0; view_wallet->import_key_images(key_images.second, key_images.first, spent, unspent, false); @@ -1782,7 +1782,7 @@ uint64_t WalletImpl::estimateTransactionFee(const std::vector<std::pair<std::str m_wallet->use_fork_rules(HF_VERSION_CLSAG, 0), m_wallet->use_fork_rules(HF_VERSION_BULLETPROOF_PLUS, 0), m_wallet->use_fork_rules(HF_VERSION_VIEW_TAGS, 0), - m_wallet->get_base_fee(), + m_wallet->get_base_fee(priority), m_wallet->get_fee_quantization_mask()); } diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index ec2d7e9b3..4ac672287 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -39,6 +39,7 @@ #include <boost/thread/thread.hpp> #include <boost/thread/condition_variable.hpp> +class WalletApiAccessorTest; namespace Monero { class TransactionHistoryImpl; @@ -248,6 +249,7 @@ private: friend class AddressBookImpl; friend class SubaddressImpl; friend class SubaddressAccountImpl; + friend class ::WalletApiAccessorTest; std::unique_ptr<tools::wallet2> m_wallet; mutable boost::mutex m_statusMutex; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 53a01914d..9e99cac83 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -13331,9 +13331,7 @@ size_t wallet2::import_outputs(const std::tuple<uint64_t, uint64_t, std::vector< THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Imported outputs omit more outputs that we know of"); - THROW_WALLET_EXCEPTION_IF(offset >= num_outputs, error::wallet_internal_error, - "Offset is larger than total outputs"); - THROW_WALLET_EXCEPTION_IF(output_array.size() > num_outputs - offset, error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset + output_array.size() > num_outputs, error::wallet_internal_error, "Offset is larger than total outputs"); const size_t original_size = m_transfers.size(); @@ -13413,9 +13411,7 @@ size_t wallet2::import_outputs(const std::tuple<uint64_t, uint64_t, std::vector< THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Imported outputs omit more outputs that we know of. Try using export_outputs all."); - THROW_WALLET_EXCEPTION_IF(offset >= num_outputs, error::wallet_internal_error, - "Offset is larger than total outputs"); - THROW_WALLET_EXCEPTION_IF(output_array.size() > num_outputs - offset, error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset + output_array.size() > num_outputs, error::wallet_internal_error, "Offset is larger than total outputs"); const size_t original_size = m_transfers.size(); diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 8474fb568..06910ebbb 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -155,8 +155,7 @@ bool wallet2::search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads const uint8_t major_version = hashing_blob[0]; if (major_version >= RX_BLOCK_VERSION) { - const int miners = 1; - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data, miners, 0); + crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data); } else { |