diff options
54 files changed, 582 insertions, 118 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c28a71ce..c3c552dd8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,8 +27,8 @@ jobs: - uses: actions/cache@v2 with: path: /Users/runner/Library/Caches/ccache - key: ccache-macos-build-${{ github.sha }} - restore-keys: ccache-macos-build- + key: ccache-${{ runner.os }}-build-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build- - name: install dependencies run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache - name: build @@ -52,12 +52,12 @@ jobs: - uses: actions/cache@v2 with: path: C:\Users\runneradmin\.ccache - key: ccache-windows-build-${{ github.sha }} - restore-keys: ccache-windows-build- + key: ccache-${{ runner.os }}-build-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build- - uses: eine/setup-msys2@v2 with: update: true - install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git - name: build run: | ccache --max-size=150M @@ -81,8 +81,8 @@ jobs: - uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-build-${{ matrix.os }}-${{ github.sha }} - restore-keys: ccache-ubuntu-build-${{ matrix.os }} + key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }} - name: remove bundled boost run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf @@ -106,8 +106,8 @@ jobs: - uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-libwallet-${{ github.sha }} - restore-keys: ccache-ubuntu-libwallet- + key: ccache-${{ runner.os }}-libwallet-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-libwallet- - name: remove bundled boost run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf @@ -136,8 +136,8 @@ jobs: uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-build-ubuntu-latest-${{ github.sha }} - restore-keys: ccache-ubuntu-build-ubuntu-latest + key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }} + restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest - name: remove bundled boost run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf diff --git a/.gitmodules b/.gitmodules index 3cf831bcd..721cce3b4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "external/unbound"] - path = external/unbound - url = https://github.com/monero-project/unbound - branch = monero [submodule "external/miniupnp"] path = external/miniupnp url = https://github.com/miniupnp/miniupnp diff --git a/CMakeLists.txt b/CMakeLists.txt index 592dfffe4..52638c064 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -347,7 +347,6 @@ if(NOT MANUAL_SUBMODULES) message(STATUS "Checking submodules") check_submodule(external/miniupnp) - check_submodule(external/unbound) check_submodule(external/rapidjson) check_submodule(external/trezor-common) check_submodule(external/randomx) @@ -675,8 +674,7 @@ include_directories("${CMAKE_CURRENT_BINARY_DIR}/translations") add_subdirectory(external) # Final setup for libunbound -include_directories(${UNBOUND_INCLUDE}) -link_directories(${UNBOUND_LIBRARY_DIRS}) +include_directories(${UNBOUND_INCLUDE_DIR}) # Final setup for easylogging++ include_directories(${EASYLOGGING_INCLUDE}) @@ -202,7 +202,7 @@ Install all dependencies at once on macOS with the provided Brewfile: ``` brew update && brew bundle --file=contrib/brew/Brewfile ``` FreeBSD 12.1 one-liner required to build dependencies: -```pkg install git gmake cmake pkgconf boost-libs libzmq4 libsodium``` +```pkg install git gmake cmake pkgconf boost-libs libzmq4 libsodium unbound``` ### Cloning the repository @@ -399,13 +399,13 @@ application. To build for 64-bit Windows: ```bash - pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi + pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-unbound ``` To build for 32-bit Windows: ```bash - pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi + pacman -S mingw-w64-i686-toolchain make mingw-w64-i686-cmake mingw-w64-i686-boost mingw-w64-i686-openssl mingw-w64-i686-zeromq mingw-w64-i686-libsodium mingw-w64-i686-hidapi mingw-w64-i686-unbound ``` * Open the MingW shell via `MinGW-w64-Win64 Shell` shortcut on 64-bit Windows diff --git a/contrib/depends/packages/expat.mk b/contrib/depends/packages/expat.mk index d73a5e307..1e1b9dbb8 100644 --- a/contrib/depends/packages/expat.mk +++ b/contrib/depends/packages/expat.mk @@ -1,8 +1,8 @@ package=expat -$(package)_version=2.2.4 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_4 +$(package)_version=2.4.1 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_4_1 $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd372019e +$(package)_sha256_hash=2f9b6a580b94577b150a7d5617ad4643a4301a6616ff459307df3e225bcfbf40 define $(package)_set_vars $(package)_config_opts=--enable-static diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index 6fbcc3466..90c63e821 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -1,8 +1,8 @@ package=ldns -$(package)_version=1.6.17 -$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/ +$(package)_version=1.7.1 +$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd +$(package)_sha256_hash=8ac84c16bdca60e710eea75782356f3ac3b55680d40e1530d7cea474ac208229 $(package)_dependencies=openssl define $(package)_set_vars diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index d80775b22..16c232b41 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -59,11 +59,11 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) -j1 build_libs libcrypto.pc libssl.pc openssl.pc + $(MAKE) build_libs endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw + $(MAKE) DESTDIR=$($(package)_staging_dir) install_sw endef define $(package)_postprocess_cmds diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 95b23a37e..56ce425bb 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,4 +1,4 @@ -packages:=boost openssl zeromq libiconv +packages:=boost openssl zeromq libiconv expat ldns unbound native_packages := native_ccache diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk index 733a7f232..2d870d63f 100644 --- a/contrib/depends/packages/unbound.mk +++ b/contrib/depends/packages/unbound.mk @@ -1,12 +1,12 @@ package=unbound -$(package)_version=1.6.8 -$(package)_download_path=https://www.unbound.net/downloads/ +$(package)_version=1.13.2 +$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49 +$(package)_sha256_hash=0a13b547f3b92a026b5ebd0423f54c991e5718037fd9f72445817f6a040e1a83 $(package)_dependencies=openssl expat ldns define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads + $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only $(package)_config_opts_linux=--with-pic $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix) $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread" diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index 383b88f31..a87b9c058 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -24,6 +24,9 @@ SET(Readline_INCLUDE_DIR @prefix@/include) SET(Readline_LIBRARY @prefix@/lib/libreadline.a) SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a) +SET(UNBOUND_INCLUDE_DIR @prefix@/include) +SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a) + SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index 19bdf4ff0..0f4a28c99 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -668,7 +668,7 @@ namespace net_utils // Cross-origin resource sharing if(m_query_info.m_header_info.m_origin.size()) { - if (std::binary_search(m_config.m_access_control_origins.begin(), m_config.m_access_control_origins.end(), m_query_info.m_header_info.m_origin)) + if (std::binary_search(m_config.m_access_control_origins.begin(), m_config.m_access_control_origins.end(), "*") || std::binary_search(m_config.m_access_control_origins.begin(), m_config.m_access_control_origins.end(), m_query_info.m_header_info.m_origin)) { buf += "Access-Control-Allow-Origin: "; buf += m_query_info.m_header_info.m_origin; diff --git a/contrib/epee/include/stats.inl b/contrib/epee/include/stats.inl index 5a5cd0b93..70c127be7 100644 --- a/contrib/epee/include/stats.inl +++ b/contrib/epee/include/stats.inl @@ -1,6 +1,7 @@ #include <math.h> #include <limits> #include <algorithm> +#include "misc_language.h" #include "stats.h" enum @@ -86,7 +87,7 @@ Tpod Stats<T, Tpod>::get_median() const } else { - median = (sorted[(sorted.size() - 1) / 2] + sorted[sorted.size() / 2]) / 2; + median = epee::misc_utils::get_mid(sorted[(sorted.size() - 1) / 2], sorted[sorted.size() / 2]); } set_cached(bit_median); } diff --git a/docs/ZMQ.md b/docs/ZMQ.md index 9128ff2ad..a7006f486 100644 --- a/docs/ZMQ.md +++ b/docs/ZMQ.md @@ -25,6 +25,8 @@ allows for filtering on: (1) format, (2) context, and (3) event. Includes previously unseen transactions in a block but _not_ the `miner_tx`. Does not "re-publish" after a reorg. Includes `do_not_relay` transactions. + * `miner_data` - provides the necessary data to create a custom block template + Available only in the `full` context. The subscription topics are formatted as `format-context-event`, with prefix matching supported by both Monero and ZMQ. The `format`, `context` and `event` diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 7ae4ba750..5b7f69a56 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -55,26 +55,12 @@ set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE) find_package(Unbound) -if(NOT UNBOUND_INCLUDE_DIR OR STATIC) - # NOTE: If STATIC is true, CMAKE_FIND_LIBRARY_SUFFIXES has been reordered. - # unbound has config tests which used OpenSSL libraries, so -ldl may need to - # be set in this case. - # The unbound CMakeLists.txt can set it, since it's also needed for the - # static OpenSSL libraries set up there after with target_link_libraries. - add_subdirectory(unbound) - - set(UNBOUND_STATIC true PARENT_SCOPE) - set(UNBOUND_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/unbound/libunbound" PARENT_SCOPE) - set(UNBOUND_LIBRARY "unbound" PARENT_SCOPE) - set(UNBOUND_LIBRARY_DIRS "${LIBEVENT2_LIBDIR}" PARENT_SCOPE) +if(NOT UNBOUND_INCLUDE_DIR) + die("Could not find libunbound") else() message(STATUS "Found libunbound include (unbound.h) in ${UNBOUND_INCLUDE_DIR}") if(UNBOUND_LIBRARIES) - message(STATUS "Found libunbound shared library") - set(UNBOUND_STATIC false PARENT_SCOPE) - set(UNBOUND_INCLUDE ${UNBOUND_INCLUDE_DIR} PARENT_SCOPE) - set(UNBOUND_LIBRARY ${UNBOUND_LIBRARIES} PARENT_SCOPE) - set(UNBOUND_LIBRARY_DIRS "" PARENT_SCOPE) + message(STATUS "Found libunbound library") else() die("Found libunbound includes, but could not find libunbound library. Please make sure you have installed libunbound or libunbound-dev or the equivalent") endif() diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index c4a88339f..b983a796c 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -2026,6 +2026,7 @@ class TypedConfigurations : public base::threading::ThreadSafe { ELPP_INTERNAL_ERROR("Unable to get configuration [" << confName << "] for level [" << LevelHelper::convertToString(level) << "]" << std::endl << "Please ensure you have properly configured logger.", false); + throw; // The exception has to be rethrown, to abort a branch leading to UB. } } return it->second; diff --git a/external/unbound b/external/unbound deleted file mode 160000 -Subproject 0f6c0579d66b65f86066e30e7876105ba2775ef diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 8e427b6b8..99d9bd8bf 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -99,7 +99,7 @@ monero_add_library(common target_link_libraries(common PUBLIC cncrypto - ${UNBOUND_LIBRARY} + ${UNBOUND_LIBRARIES} ${LIBUNWIND_LIBRARIES} ${Boost_DATE_TIME_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 6394a7071..d111f6f32 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -49,6 +49,7 @@ #include "misc_language.h" #include "ringct/rctTypes.h" #include "device/device.hpp" +#include "cryptonote_basic/fwd.h" namespace cryptonote { diff --git a/src/cryptonote_basic/events.h b/src/cryptonote_basic/events.h index 6c6742215..3417ece8c 100644 --- a/src/cryptonote_basic/events.h +++ b/src/cryptonote_basic/events.h @@ -41,6 +41,8 @@ namespace cryptonote { cryptonote::transaction tx; crypto::hash hash; + uint64_t blob_size; + uint64_t weight; bool res; //!< Listeners must ignore `tx` when this is false. }; } diff --git a/src/cryptonote_basic/fwd.h b/src/cryptonote_basic/fwd.h index d54223461..901ad151b 100644 --- a/src/cryptonote_basic/fwd.h +++ b/src/cryptonote_basic/fwd.h @@ -33,4 +33,5 @@ namespace cryptonote struct block; class transaction; struct txpool_event; + struct tx_block_template_backlog_entry; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index f0e6794b9..3d0e81af1 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1238,6 +1238,12 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + crypto::hash prev_id; + if (!get_block_hash(alt_chain.back().bl, prev_id)) + MERROR("Failed to get block hash of an alternative chain's tip"); + else + send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + for (const auto& notifier : m_block_notifiers) { std::size_t notify_height = split_height; @@ -1784,6 +1790,30 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //------------------------------------------------------------------ +bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + prev_id = m_db->top_block_hash(&height); + ++height; + + major_version = m_hardfork->get_ideal_version(height); + + seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + difficulty = get_difficulty_for_next_block(); + median_weight = m_current_block_cumul_weight_median; + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + + m_tx_pool.get_block_template_backlog(tx_backlog); + + return true; +} +//------------------------------------------------------------------ // for an alternate chain, get the timestamps from the main chain to complete // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const @@ -4362,6 +4392,7 @@ leave: get_difficulty_for_next_block(); // just to cache it invalidate_block_template_cache(); + send_miner_notifications(id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); @@ -5270,7 +5301,7 @@ void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint m_max_prepare_blocks_threads = maxthreads; } -void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)>&& notify) +void Blockchain::add_block_notify(BlockNotifyCallback&& notify) { if (notify) { @@ -5279,6 +5310,15 @@ void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span } } +void Blockchain::add_miner_notify(MinerNotifyCallback&& notify) +{ + if (notify) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_miner_notifiers.push_back(std::move(notify)); + } +} + void Blockchain::safesyncmode(const bool onoff) { /* all of this is no-op'd if the user set a specific @@ -5531,6 +5571,33 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_valid = true; } +void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins) +{ + if (m_miner_notifiers.empty()) + return; + + const uint64_t height = m_db->height(); + const uint8_t major_version = m_hardfork->get_ideal_version(height); + const difficulty_type diff = get_difficulty_for_next_block(); + const uint64_t median_weight = m_current_block_cumul_weight_median; + + crypto::hash seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + std::vector<tx_block_template_backlog_entry> tx_backlog; + m_tx_pool.get_block_template_backlog(tx_backlog); + + for (const auto& notifier : m_miner_notifiers) + { + notifier(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + } +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index a0e7967de..9afbfbc2d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -90,6 +90,9 @@ namespace cryptonote */ typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback; + typedef boost::function<void(uint64_t /* height */, epee::span<const block> /* blocks */)> BlockNotifyCallback; + typedef boost::function<void(uint8_t /* major_version */, uint64_t /* height */, const crypto::hash& /* prev_id */, const crypto::hash& /* seed_hash */, difficulty_type /* diff */, uint64_t /* median_weight */, uint64_t /* already_generated_coins */, const std::vector<tx_block_template_backlog_entry>& /* tx_backlog */)> MinerNotifyCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -371,6 +374,22 @@ namespace cryptonote bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @brief gets data required to create a block template and start mining on it + * + * @param major_version current hardfork version + * @param height current blockchain height + * @param prev_id hash of the top block + * @param seed_hash seed hash used for RandomX initialization + * @param difficulty current mining difficulty + * @param median_weight current median block weight + * @param already_generated_coins current emission + * @param tx_backlog transactions in mempool ready to be mined + * + * @return true if block template filled in successfully, else false + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief checks if a block is known about with a given hash * * This function checks the main chain, alternate chains, and invalid blocks @@ -775,7 +794,14 @@ namespace cryptonote * * @param notify the notify object to call at every new block */ - void add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)> &¬ify); + void add_block_notify(BlockNotifyCallback&& notify); + + /** + * @brief sets a miner notify object to call for every new block + * + * @param notify the notify object to call at every new block + */ + void add_miner_notify(MinerNotifyCallback&& notify); /** * @brief sets a reorg notify object to call for every reorg @@ -1157,7 +1183,8 @@ namespace cryptonote the callable object has a single `std::shared_ptr` or `std::weap_ptr` internally. Whereas, the libstdc++ `std::function` will allocate. */ - std::vector<boost::function<void(std::uint64_t, epee::span<const block>)>> m_block_notifiers; + std::vector<BlockNotifyCallback> m_block_notifiers; + std::vector<MinerNotifyCallback> m_miner_notifiers; std::shared_ptr<tools::Notify> m_reorg_notify; // for prepare_handle_incoming_blocks @@ -1537,5 +1564,13 @@ namespace cryptonote * At some point, may be used to push an update to miners */ void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t seed_height, const crypto::hash &seed_hash, uint64_t pool_cookie); + + /** + * @brief sends new block notifications to ZMQ `miner_data` subscribers + * + * @param prev_id hash of new blockchain tip + * @param already_generated_coins total coins mined by the network so far + */ + void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 1da14221a..4c6536318 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -388,6 +388,7 @@ namespace cryptonote m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); + if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) MWARNING(arg_fluffy_blocks.name << " is obsolete, it is now default"); @@ -460,7 +461,7 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */) + bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */, bool allow_dns) { start_time = std::time(nullptr); @@ -471,6 +472,7 @@ namespace cryptonote } bool r = handle_command_line(vm); CHECK_AND_ASSERT_MES(r, false, "Failed to handle command line"); + m_disable_dns_checkpoints |= not allow_dns; std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); bool db_salvage = command_line::get_arg(vm, cryptonote::arg_db_salvage) != 0; @@ -697,7 +699,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(update_checkpoints(skip_dns_checkpoints), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); // DNS versions checking - if (check_updates_string == "disabled") + if (check_updates_string == "disabled" || not allow_dns) check_updates_level = UPDATES_DISABLED; else if (check_updates_string == "notify") check_updates_level = UPDATES_NOTIFY; @@ -1063,8 +1065,9 @@ namespace cryptonote if (already_have[i]) continue; - const uint64_t weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], tx_relay, relayed); + results[i].blob_size = it->blob.size(); + results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, results[i].weight, tvc[i], tx_relay, relayed); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} @@ -1403,6 +1406,11 @@ namespace cryptonote return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //----------------------------------------------------------------------------------------------- + bool core::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) + { + return m_blockchain_storage.get_miner_data(major_version, height, prev_id, seed_hash, difficulty, median_weight, already_generated_coins, tx_backlog); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8478049f9..d2bffdaee 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -237,6 +237,13 @@ namespace cryptonote virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @copydoc Blockchain::get_miner_data + * + * @note see Blockchain::get_miner_data + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief called when a transaction is relayed. * @note Should only be invoked from `levin_notify`. */ @@ -276,10 +283,11 @@ namespace cryptonote * @param vm command line parameters * @param test_options configuration options for testing * @param get_checkpoints if set, will be called to get checkpoints data, must return checkpoints data pointer and size or nullptr if there ain't any checkpoints for specific network type + * @param allow_dns whether or not to allow DNS requests * * @return false if one of the init steps fails, otherwise true */ - bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr); + bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr, bool allow_dns = true); /** * @copydoc Blockchain::reset_and_set_genesis_block diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 73cdd31cd..06412d6bf 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -108,6 +108,15 @@ namespace cryptonote }; //--------------------------------------------------------------- + + struct tx_block_template_backlog_entry + { + crypto::hash id; + uint64_t weight; + uint64_t fee; + }; + + //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index a7e96e23a..84605d6f5 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -913,6 +913,32 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + void tx_memory_pool::get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); + txpool_tx_meta_t tmp_meta; + m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ + transaction tx; + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) + { + MERROR("Failed to parse tx from txpool"); + // continue + return true; + } + tx.set_hash(txid); + + tmp_meta = meta; + + if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx)) + backlog.push_back({txid, meta.weight, meta.fee}); + + return true; + }, true, category); + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1222,11 +1248,11 @@ namespace cryptonote return ret; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref& txblob, transaction &tx) const { - struct transction_parser + struct transaction_parser { - transction_parser(const cryptonote::blobdata &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} + transaction_parser(const cryptonote::blobdata_ref &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} cryptonote::transaction &operator()() { if (!parsed) @@ -1238,7 +1264,7 @@ namespace cryptonote } return tx; } - const cryptonote::blobdata &txblob; + const cryptonote::blobdata_ref &txblob; const crypto::hash &txid; transaction &tx; bool parsed; @@ -1289,6 +1315,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata& txblob, transaction &tx) const + { + return is_transaction_ready_to_go(txd, txid, cryptonote::blobdata_ref{txblob.data(), txblob.size()}, tx); + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::have_key_images(const std::unordered_set<crypto::key_image>& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index ab2a57ea2..80b38c51d 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -266,6 +266,15 @@ namespace cryptonote void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive = false) const; /** + * @brief get (hash, weight, fee) for all transactions in the pool - the minimum required information to create a block template + * + * @param backlog return-by-reference that data + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes + * + */ + void get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive = false) const; + + /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics @@ -540,6 +549,7 @@ namespace cryptonote * * @return true if the transaction is good to go, otherwise false */ + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref &txblob, transaction&tx) const; bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction&tx) const; /** diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 6c3e163e6..a988fe25f 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -96,6 +96,16 @@ namespace daemon_args , 0 }; + const command_line::arg_descriptor<std::string> arg_proxy = { + "proxy", + "Network communication through proxy: <socks-ip:port> i.e. \"127.0.0.1:9050\"", + "", + }; + const command_line::arg_descriptor<bool> arg_proxy_allow_dns_leaks = { + "proxy-allow-dns-leaks", + "Allow DNS leaks outside of proxy", + false, + }; const command_line::arg_descriptor<bool> arg_public_node = { "public-node" , "Allow other users to use the node as a remote (restricted RPC mode, view-only commands) and advertise it over P2P" diff --git a/src/daemon/core.h b/src/daemon/core.h index 804d7474d..0811cf420 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -32,6 +32,7 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "misc_log_ex.h" +#include "daemon/command_line_args.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -66,7 +67,14 @@ public: #else const cryptonote::GetCheckpointsCallback& get_checkpoints = nullptr; #endif - if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints)) + + if (command_line::is_arg_defaulted(vm, daemon_args::arg_proxy) && command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks)) { + MLOG_RED(el::Level::Warning, "--" << daemon_args::arg_proxy_allow_dns_leaks.name << " is enabled, but --" + << daemon_args::arg_proxy.name << " is not specified."); + } + + const bool allow_dns = command_line::is_arg_defaulted(vm, daemon_args::arg_proxy) || command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks); + if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints, allow_dns)) { throw std::runtime_error("Failed to initialize core"); } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 99430b2b0..3f1885423 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -120,6 +120,7 @@ public: if (shared) { core.get().get_blockchain_storage().add_block_notify(cryptonote::listener::zmq_pub::chain_main{shared}); + core.get().get_blockchain_storage().add_miner_notify(cryptonote::listener::zmq_pub::miner_data{shared}); core.get().set_txpool_listener(cryptonote::listener::zmq_pub::txpool_add{shared}); } } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index d413906df..70aec5538 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -152,6 +152,8 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); + command_line::add_arg(core_settings, daemon_args::arg_proxy); + command_line::add_arg(core_settings, daemon_args::arg_proxy_allow_dns_leaks); command_line::add_arg(core_settings, daemon_args::arg_public_node); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index f68efccc2..38862c017 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -33,6 +33,7 @@ #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "p2p/net_node.h" #include "daemon/protocol.h" +#include "daemon/command_line_args.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -61,7 +62,7 @@ public: { //initialize objects MGINFO("Initializing p2p server..."); - if (!m_server.init(vm)) + if (!m_server.init(vm, command_line::get_arg(vm, daemon_args::arg_proxy), command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks))) { throw std::runtime_error("Failed to initialize p2p server."); } diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index af48bcc45..bff7dc449 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -62,7 +62,7 @@ public: { MGINFO("Initializing " << m_description << " RPC server..."); - if (!m_server.init(vm, restricted, port, allow_rpc_payment)) + if (!m_server.init(vm, restricted, port, allow_rpc_payment, command_line::get_arg(vm, daemon_args::arg_proxy))) { throw std::runtime_error("Failed to initialize " + m_description + " RPC server."); } diff --git a/src/device/device.hpp b/src/device/device.hpp index 582eb2242..6005e157d 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -91,7 +91,6 @@ namespace hw { public: device(): mode(NONE) {} - device(const device &hwdev) {} virtual ~device() {} explicit virtual operator bool() const = 0; diff --git a/src/net/socks.h b/src/net/socks.h index 739c972ab..506b53195 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -201,6 +201,13 @@ namespace socks std::shared_ptr<client> self_; void operator()(boost::system::error_code error = boost::system::error_code{}); }; + + //! Calls `async_close` on `self` at destruction. NOP if `nullptr`. + struct close_on_exit + { + std::shared_ptr<client> self; + ~close_on_exit() { async_close{std::move(self)}(); } + }; }; template<typename Handler> diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 84cc1581e..d9050200a 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -94,6 +94,9 @@ namespace case net::i2p_address::get_type_id(): set = client->set_connect_command(remote.as<net::i2p_address>()); break; + case epee::net_utils::ipv4_network_address::get_type_id(): + set = client->set_connect_command(remote.as<epee::net_utils::ipv4_network_address>()); + break; default: MERROR("Unsupported network address in socks_connect"); return false; @@ -339,6 +342,7 @@ namespace nodetool } }; + net::socks::client::close_on_exit close_client{}; boost::unique_future<client_result> socks_result{}; { boost::promise<client_result> socks_promise{}; @@ -347,6 +351,7 @@ namespace nodetool auto client = net::socks::make_connect_client( boost::asio::ip::tcp::socket{service}, net::socks::version::v4a, notify{std::move(socks_promise)} ); + close_client.self = client; if (!start_socks(std::move(client), proxy, remote)) return boost::none; } @@ -368,7 +373,10 @@ namespace nodetool { auto result = socks_result.get(); if (!result.first) + { + close_client.self.reset(); return {std::move(result.second)}; + } MERROR("Failed to make socks connection to " << remote.str() << " (via " << proxy << "): " << result.first.message()); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index f2888674b..9e64121be 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -259,6 +259,7 @@ namespace nodetool m_offline(false), is_closing(false), m_network_id(), + m_enable_dns_seed_nodes(true), max_connections(1) {} virtual ~node_server(); @@ -267,7 +268,7 @@ namespace nodetool bool run(); network_zone& add_zone(epee::net_utils::zone zone); - bool init(const boost::program_options::variables_map& vm); + bool init(const boost::program_options::variables_map& vm, const std::string& proxy = {}, bool proxy_dns_leaks_allowed = {}); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listening_port;} @@ -516,6 +517,7 @@ namespace nodetool epee::net_utils::ssl_support_t m_ssl_support; + bool m_enable_dns_seed_nodes; bool m_enable_dns_blocklist; uint32_t max_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index cfeac3d37..ac65a57c1 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -741,6 +741,12 @@ namespace nodetool { return get_ip_seed_nodes(); } + if (!m_enable_dns_seed_nodes) + { + // TODO: a domain can be set through socks, so that the remote side does the lookup for the DNS seed nodes. + m_fallback_seed_nodes_added.test_and_set(); + return get_ip_seed_nodes(); + } std::set<std::string> full_addrs; @@ -880,10 +886,21 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) + bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm, const std::string& proxy, bool proxy_dns_leaks_allowed) { bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); + if (proxy.size()) + { + const auto endpoint = net::get_tcp_endpoint(proxy); + CHECK_AND_ASSERT_MES(endpoint, false, "Failed to parse proxy: " << proxy << " - " << endpoint.error()); + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + public_zone.m_connect = &socks_connect; + public_zone.m_proxy_address = *endpoint; + public_zone.m_can_pingback = false; + m_enable_dns_seed_nodes &= proxy_dns_leaks_allowed; + m_enable_dns_blocklist &= proxy_dns_leaks_allowed; + } if (m_nettype == cryptonote::TESTNET) { diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 620f7d0dd..784c90a4e 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -449,7 +449,6 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str STEP = STEP ? STEP : 192; MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); - static constexpr unsigned int mask = (1<<STRAUS_C)-1; std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; ge_cached cached; ge_p1p1 p1; @@ -483,6 +482,7 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str memcpy(bytes33, data[j].scalar.bytes, 32); bytes33[32] = 0; bytes = bytes33; + static constexpr unsigned int mask = (1<<STRAUS_C)-1; for (size_t i = 0; i < 256; ++i) digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; #else diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 8d8a68efb..942bfce0a 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -242,11 +242,11 @@ namespace cryptonote auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; - m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, proxy)); + m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, m_bootstrap_daemon_proxy.empty() ? proxy : m_bootstrap_daemon_proxy)); } else { - m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, proxy)); + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, m_bootstrap_daemon_proxy.empty() ? proxy : m_bootstrap_daemon_proxy)); } m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; @@ -264,8 +264,10 @@ namespace cryptonote , const bool restricted , const std::string& port , bool allow_rpc_payment + , const std::string& proxy ) { + m_bootstrap_daemon_proxy = proxy; m_restricted = restricted; m_net_server.set_threads_prefix("RPC"); m_net_server.set_connection_filter(&m_p2p); @@ -1861,6 +1863,43 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + if(!check_core_ready()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy"; + return false; + } + + crypto::hash prev_id, seed_hash; + difficulty_type difficulty; + + std::vector<tx_block_template_backlog_entry> tx_backlog; + if (!m_core.get_miner_data(res.major_version, res.height, prev_id, seed_hash, difficulty, res.median_weight, res.already_generated_coins, tx_backlog)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: failed to get miner data"; + LOG_ERROR("Failed to get miner data"); + return false; + } + + res.tx_backlog.clear(); + res.tx_backlog.reserve(tx_backlog.size()); + + for (const auto& entry : tx_backlog) + { + res.tx_backlog.emplace_back(COMMAND_RPC_GETMINERDATA::response::tx_backlog_entry{string_tools::pod_to_hex(entry.id), entry.weight, entry.fee}); + } + + res.prev_id = string_tools::pod_to_hex(prev_id); + res.seed_hash = string_tools::pod_to_hex(seed_hash); + res.difficulty = cryptonote::hex(difficulty); + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(add_aux_pow); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index b21e43ab0..84b14383a 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -91,7 +91,8 @@ namespace cryptonote const boost::program_options::variables_map& vm, const bool restricted, const std::string& port, - bool allow_rpc_payment + bool allow_rpc_payment, + const std::string& proxy = {} ); network_type nettype() const { return m_core.get_nettype(); } @@ -147,6 +148,7 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("get_miner_data", on_getminerdata, COMMAND_RPC_GETMINERDATA) MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) @@ -228,6 +230,7 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); @@ -289,6 +292,7 @@ private: nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_p2p; boost::shared_mutex m_bootstrap_daemon_mutex; std::unique_ptr<bootstrap_daemon> m_bootstrap_daemon; + std::string m_bootstrap_daemon_proxy; bool m_should_use_bootstrap_daemon; std::chrono::system_clock::time_point m_bootstrap_height_check_time; bool m_was_bootstrap_ever_used; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index ff8c98b98..1dbfc83a7 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 7 +#define CORE_RPC_VERSION_MINOR 8 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -940,6 +940,56 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_GETMINERDATA + { + struct request_t: public rpc_request_base + { + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + uint8_t major_version; + uint64_t height; + std::string prev_id; + std::string seed_hash; + std::string difficulty; + uint64_t median_weight; + uint64_t already_generated_coins; + + struct tx_backlog_entry + { + std::string id; + uint64_t weight; + uint64_t fee; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(weight) + KV_SERIALIZE(fee) + END_KV_SERIALIZE_MAP() + }; + + std::vector<tx_backlog_entry> tx_backlog; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(major_version) + KV_SERIALIZE(height) + KV_SERIALIZE(prev_id) + KV_SERIALIZE(seed_hash) + KV_SERIALIZE(difficulty) + KV_SERIALIZE(median_weight) + KV_SERIALIZE(already_generated_coins) + KV_SERIALIZE(tx_backlog) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_ADD_AUX_POW { struct aux_pow_t diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp index eac530968..074b55207 100644 --- a/src/rpc/zmq_pub.cpp +++ b/src/rpc/zmq_pub.cpp @@ -48,6 +48,8 @@ #include "cryptonote_basic/events.h" #include "misc_log_ex.h" #include "serialization/json_object.h" +#include "ringct/rctTypes.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.zmq" @@ -57,6 +59,7 @@ namespace constexpr const char txpool_signal[] = "tx_signal"; using chain_writer = void(epee::byte_stream&, std::uint64_t, epee::span<const cryptonote::block>); + using miner_writer = void(epee::byte_stream&, uint8_t, uint64_t, const crypto::hash&, const crypto::hash&, cryptonote::difficulty_type, uint64_t, uint64_t, const std::vector<cryptonote::tx_block_template_backlog_entry>&); using txpool_writer = void(epee::byte_stream&, epee::span<const cryptonote::txpool_event>); template<typename F> @@ -116,13 +119,30 @@ namespace const epee::span<const cryptonote::block> blocks; }; + //! Object for miner data serialization + struct miner_data + { + uint8_t major_version; + uint64_t height; + const crypto::hash& prev_id; + const crypto::hash& seed_hash; + cryptonote::difficulty_type diff; + uint64_t median_weight; + uint64_t already_generated_coins; + const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog; + }; + //! Object for "minimal" tx serialization struct minimal_txpool { const cryptonote::transaction& tx; + crypto::hash hash; + uint64_t blob_size; + uint64_t weight; + uint64_t fee; }; - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain& self) { namespace adapt = boost::adaptors; @@ -143,19 +163,27 @@ namespace dest.EndObject(); } - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const miner_data& self) { - crypto::hash id{}; - std::size_t blob_size = 0; - if (!get_transaction_hash(self.tx, id, blob_size)) - { - MERROR("ZMQ/Pub failure: get_transaction_hash"); - return; - } + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, major_version, self.major_version); + INSERT_INTO_JSON_OBJECT(dest, height, self.height); + INSERT_INTO_JSON_OBJECT(dest, prev_id, self.prev_id); + INSERT_INTO_JSON_OBJECT(dest, seed_hash, self.seed_hash); + INSERT_INTO_JSON_OBJECT(dest, difficulty, cryptonote::hex(self.diff)); + INSERT_INTO_JSON_OBJECT(dest, median_weight, self.median_weight); + INSERT_INTO_JSON_OBJECT(dest, already_generated_coins, self.already_generated_coins); + INSERT_INTO_JSON_OBJECT(dest, tx_backlog, self.tx_backlog); + dest.EndObject(); + } + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool& self) + { dest.StartObject(); - INSERT_INTO_JSON_OBJECT(dest, id, id); - INSERT_INTO_JSON_OBJECT(dest, blob_size, blob_size); + INSERT_INTO_JSON_OBJECT(dest, id, self.hash); + INSERT_INTO_JSON_OBJECT(dest, blob_size, self.blob_size); + INSERT_INTO_JSON_OBJECT(dest, weight, self.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, self.fee); dest.EndObject(); } @@ -169,6 +197,11 @@ namespace json_pub(buf, minimal_chain{height, blocks}); } + void json_miner_data(epee::byte_stream& buf, uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, cryptonote::difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog) + { + json_pub(buf, miner_data{major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog}); + } + // boost::adaptors are in place "views" - no copy/move takes place // moving transactions (via sort, etc.), is expensive! @@ -187,7 +220,7 @@ namespace namespace adapt = boost::adaptors; const auto to_minimal_tx = [](const cryptonote::txpool_event& event) { - return minimal_txpool{event.tx}; + return minimal_txpool{event.tx, event.hash, event.blob_size, event.weight, cryptonote::get_tx_fee(event.tx)}; }; json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_minimal_tx))); } @@ -198,6 +231,11 @@ namespace {u8"json-minimal-chain_main", json_minimal_chain} }}; + constexpr const std::array<context<miner_writer>, 1> miner_contexts = + {{ + {u8"json-full-miner_data", json_miner_data}, + }}; + constexpr const std::array<context<txpool_writer>, 2> txpool_contexts = {{ {u8"json-full-txpool_add", json_full_txpool}, @@ -321,6 +359,7 @@ namespace cryptonote { namespace listener zmq_pub::zmq_pub(void* context) : relay_(), chain_subs_{{0}}, + miner_subs_{{0}}, txpool_subs_{{0}}, sync_() { @@ -328,6 +367,7 @@ zmq_pub::zmq_pub(void* context) throw std::logic_error{"ZMQ context cannot be NULL"}; verify_sorted(chain_contexts, "chain_contexts"); + verify_sorted(miner_contexts, "miner_contexts"); verify_sorted(txpool_contexts, "txpool_contexts"); relay_.reset(zmq_socket(context, ZMQ_PAIR)); @@ -348,22 +388,25 @@ bool zmq_pub::sub_request(boost::string_ref message) message.remove_prefix(1); const auto chain_range = get_range(chain_contexts, message); + const auto miner_range = get_range(miner_contexts, message); const auto txpool_range = get_range(txpool_contexts, message); - if (!chain_range.empty() || !txpool_range.empty()) + if (!chain_range.empty() || !miner_range.empty() || !txpool_range.empty()) { MDEBUG("Client " << (tag ? "subscribed" : "unsubscribed") << " to " << - chain_range.size() << " chain topic(s) and " << txpool_range.size() << " txpool topic(s)"); + chain_range.size() << " chain topic(s), " << miner_range.size() << " miner topic(s) and " << txpool_range.size() << " txpool topic(s)"); const boost::lock_guard<boost::mutex> lock{sync_}; switch (tag) { case 0: remove_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + remove_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); remove_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; case 1: add_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + add_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); add_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; default: @@ -436,6 +479,25 @@ std::size_t zmq_pub::send_chain_main(const std::uint64_t height, const epee::spa return 0; } +std::size_t zmq_pub::send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + boost::unique_lock<boost::mutex> guard{sync_}; + + const auto subs_copy = miner_subs_; + guard.unlock(); + + for (const std::size_t sub : subs_copy) + { + if (sub) + { + auto messages = make_pubs(subs_copy, miner_contexts, major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + guard.lock(); + return send_messages(relay_.get(), messages); + } + } + return 0; +} + std::size_t zmq_pub::send_txpool_add(std::vector<txpool_event> txes) { if (txes.empty()) @@ -466,6 +528,15 @@ void zmq_pub::chain_main::operator()(const std::uint64_t height, epee::span<cons MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); } +void zmq_pub::miner_data::operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const +{ + const std::shared_ptr<zmq_pub> self = self_.lock(); + if (self) + self->send_miner_data(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + else + MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); +} + void zmq_pub::txpool_add::operator()(std::vector<cryptonote::txpool_event> txes) const { const std::shared_ptr<zmq_pub> self = self_.lock(); diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h index 02e6b8103..c636e1d7b 100644 --- a/src/rpc/zmq_pub.h +++ b/src/rpc/zmq_pub.h @@ -39,6 +39,7 @@ #include "cryptonote_basic/fwd.h" #include "net/zmq.h" #include "span.h" +#include "cryptonote_basic/difficulty.h" namespace cryptonote { namespace listener { @@ -59,6 +60,7 @@ class zmq_pub net::zmq::socket relay_; std::deque<std::vector<txpool_event>> txes_; std::array<std::size_t, 2> chain_subs_; + std::array<std::size_t, 1> miner_subs_; std::array<std::size_t, 2> txpool_subs_; boost::mutex sync_; //!< Synchronizes counts in `*_subs_` arrays. @@ -88,6 +90,11 @@ class zmq_pub \return Number of ZMQ messages sent to relay. */ std::size_t send_chain_main(std::uint64_t height, epee::span<const cryptonote::block> blocks); + /*! Send a `ZMQ_PUB` notification for a new miner data. + Thread-safe. + \return Number of ZMQ messages sent to relay. */ + std::size_t send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog); + /*! Send a `ZMQ_PUB` notification for new tx(es) being added to the local pool. Thread-safe. \return Number of ZMQ messages sent to relay. */ @@ -100,6 +107,13 @@ class zmq_pub void operator()(std::uint64_t height, epee::span<const cryptonote::block> blocks) const; }; + //! Callable for `send_miner_data` with weak ownership to `zmq_pub` object. + struct miner_data + { + std::weak_ptr<zmq_pub> self_; + void operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const; + }; + //! Callable for `send_txpool_add` with weak ownership to `zmq_pub` object. struct txpool_add { diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 28e207ff2..b03da1edc 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -34,6 +34,7 @@ #include <type_traits> #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_core/cryptonote_tx_utils.h" // drop macro from windows.h #ifdef GetObject @@ -1411,6 +1412,27 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribu GET_FROM_JSON_OBJECT(val, dist.data.base, base); } +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry) +{ + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, id, entry.id); + INSERT_INTO_JSON_OBJECT(dest, weight, entry.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, entry.fee); + dest.EndObject(); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, entry.id, id); + GET_FROM_JSON_OBJECT(val, entry.weight, weight); + GET_FROM_JSON_OBJECT(val, entry.fee, fee); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 35ea990b3..c858faf5a 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry); +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index c56a21346..5a4cafc32 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -49,6 +49,7 @@ using namespace epee; #include "cryptonote_core/tx_sanity_check.h" #include "wallet_rpc_helpers.h" #include "wallet2.h" +#include "wallet_args.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "net/parse.h" #include "rpc/core_rpc_server_commands_defs.h" @@ -276,7 +277,7 @@ struct options { const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; - const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; + const command_line::arg_descriptor<std::string> password_file = wallet_args::arg_password_file(); const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<std::string> daemon_login = {"daemon-login", tools::wallet2::tr("Specify username[:password] for daemon RPC client"), "", true}; const command_line::arg_descriptor<std::string> daemon_ssl = {"daemon-ssl", tools::wallet2::tr("Enable SSL on daemon RPC connections: enabled|disabled|autodetect"), "autodetect"}; @@ -532,7 +533,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl boost::optional<tools::password_container> get_password(const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char*, bool)> &password_prompter, const bool verify) { - if (command_line::has_arg(vm, opts.password) && command_line::has_arg(vm, opts.password_file)) + if (command_line::has_arg(vm, opts.password) && !command_line::is_arg_defaulted(vm, opts.password_file)) { THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, tools::wallet2::tr("can't specify more than one of --password and --password-file")); } @@ -542,10 +543,11 @@ boost::optional<tools::password_container> get_password(const boost::program_opt return tools::password_container{command_line::get_arg(vm, opts.password)}; } - if (command_line::has_arg(vm, opts.password_file)) + if (!command_line::is_arg_defaulted(vm, opts.password_file)) { std::string password; - bool r = epee::file_io_utils::load_file_to_string(command_line::get_arg(vm, opts.password_file), + const auto password_file = command_line::get_arg(vm, opts.password_file); + bool r = epee::file_io_utils::load_file_to_string(password_file, password); THROW_WALLET_EXCEPTION_IF(!r, tools::error::wallet_internal_error, tools::wallet2::tr("the password file specified could not be read")); @@ -2860,7 +2862,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); const cryptonote::transaction& tx = parsed_blocks[i].block.miner_tx; const size_t n_vouts = (m_refresh_type == RefreshType::RefreshOptimizeCoinbase && tx.version < 2) ? 1 : tx.vout.size(); - tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(tx, n_vouts, txidx); }, true); + tpool.submit(&waiter, [&, n_vouts, txidx](){ geniod(tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) @@ -10261,6 +10263,38 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp const size_t num_outputs = get_num_outputs(tx.dsts, m_transfers, tx.selected_transfers); needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, num_outputs, extra.size(), bulletproof, clsag, base_fee, fee_multiplier, fee_quantization_mask); + auto try_carving_from_partial_payment = [&](uint64_t needed_fee, uint64_t available_for_fee) + { + // The check against original_output_index is to ensure the last entry in tx.dsts is really + // a partial payment. Otherwise multiple requested outputs to the same address could + // fool this logic into thinking there is a partial payment. + if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0 && tx.dsts.size() > original_output_index) + { + // we don't have enough for the fee, but we've only partially paid the current address, + // so we can take the fee from the paid amount, since we'll have to make another tx anyway + LOG_PRINT_L2("Attempting to carve tx fee " << print_money(needed_fee) << " from partial payment (first pass)"); + std::vector<cryptonote::tx_destination_entry>::iterator i; + i = std::find_if(tx.dsts.begin(), tx.dsts.end(), + [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &dsts[0].addr, sizeof(dsts[0].addr)); }); + THROW_WALLET_EXCEPTION_IF(i == tx.dsts.end(), error::wallet_internal_error, "paid address not found in outputs"); + if (i->amount > needed_fee) + { + uint64_t new_paid_amount = i->amount /*+ test_ptx.fee*/ - needed_fee; + LOG_PRINT_L2("Adjusting amount paid to " << get_account_address_as_str(m_nettype, i->is_subaddress, i->addr) << " from " << + print_money(i->amount) << " to " << print_money(new_paid_amount) << " to accommodate " << + print_money(needed_fee) << " fee"); + dsts[0].amount += i->amount - new_paid_amount; + i->amount = new_paid_amount; + test_ptx.fee = needed_fee; + available_for_fee = needed_fee; + } + } + return available_for_fee; + }; + + // Try to carve the estimated fee from the partial payment (if there is one) + available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); + uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); for (const auto &o: tx.dsts) outputs += o.amount; @@ -10286,26 +10320,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); - if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) - { - // we don't have enough for the fee, but we've only partially paid the current address, - // so we can take the fee from the paid amount, since we'll have to make another tx anyway - std::vector<cryptonote::tx_destination_entry>::iterator i; - i = std::find_if(tx.dsts.begin(), tx.dsts.end(), - [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &dsts[0].addr, sizeof(dsts[0].addr)); }); - THROW_WALLET_EXCEPTION_IF(i == tx.dsts.end(), error::wallet_internal_error, "paid address not found in outputs"); - if (i->amount > needed_fee) - { - uint64_t new_paid_amount = i->amount /*+ test_ptx.fee*/ - needed_fee; - LOG_PRINT_L2("Adjusting amount paid to " << get_account_address_as_str(m_nettype, i->is_subaddress, i->addr) << " from " << - print_money(i->amount) << " to " << print_money(new_paid_amount) << " to accommodate " << - print_money(needed_fee) << " fee"); - dsts[0].amount += i->amount - new_paid_amount; - i->amount = new_paid_amount; - test_ptx.fee = needed_fee; - available_for_fee = needed_fee; - } - } + // Try to carve the fee from the partial payment again after updating from estimate to actual + available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); if (needed_fee > available_for_fee) { diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 55058bf4e..066e98e52 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -80,6 +80,10 @@ namespace wallet_args { return {"rpc-client-secret-key", wallet_args::tr("Set RPC client secret key for RPC payments"), ""}; } + command_line::arg_descriptor<std::string> arg_password_file() + { + return {"password-file", wallet_args::tr("Wallet password file"), ""}; + } const char* tr(const char* str) { diff --git a/src/wallet/wallet_args.h b/src/wallet/wallet_args.h index 4af1b58fe..21e5f187c 100644 --- a/src/wallet/wallet_args.h +++ b/src/wallet/wallet_args.h @@ -37,6 +37,7 @@ namespace wallet_args command_line::arg_descriptor<std::string> arg_generate_from_json(); command_line::arg_descriptor<std::string> arg_wallet_file(); command_line::arg_descriptor<std::string> arg_rpc_client_secret_key(); + command_line::arg_descriptor<std::string> arg_password_file(); const char* tr(const char* str); diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index e1a06886b..4655e24cd 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -41,6 +41,7 @@ using namespace epee; #include "wallet/wallet_args.h" #include "common/command_line.h" #include "common/i18n.h" +#include "common/scoped_message_writer.h" #include "cryptonote_config.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" @@ -4525,10 +4526,12 @@ public: const auto arg_wallet_file = wallet_args::arg_wallet_file(); const auto arg_from_json = wallet_args::arg_generate_from_json(); const auto arg_rpc_client_secret_key = wallet_args::arg_rpc_client_secret_key(); + const auto arg_password_file = wallet_args::arg_password_file(); const auto wallet_file = command_line::get_arg(vm, arg_wallet_file); const auto from_json = command_line::get_arg(vm, arg_from_json); const auto wallet_dir = command_line::get_arg(vm, arg_wallet_dir); + const auto password_file = command_line::get_arg(vm, arg_password_file); const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password); const auto password_prompt = prompt_for_password ? password_prompter : nullptr; @@ -4538,6 +4541,12 @@ public: return false; } + if(!wallet_dir.empty() && !password_file.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("--password-file is not allowed in combination with --wallet-dir")); + return false; + } + if (!wallet_dir.empty()) { wal = NULL; @@ -4716,7 +4725,7 @@ int main(int argc, char** argv) { tools::wallet_rpc_server::tr("This is the RPC monero wallet. It needs to connect to a monero\ndaemon to work correctly."), desc_params, po::positional_options_description(), - [](const std::string &s, bool emphasis){ epee::set_console_color(emphasis ? epee::console_color_white : epee::console_color_default, true); std::cout << s << std::endl; if (emphasis) epee::reset_console_color(); }, + [](const std::string &s, bool emphasis){ tools::scoped_message_writer(emphasis ? epee::console_color_white : epee::console_color_default, true) << s; }, "monero-wallet-rpc.log", true ); diff --git a/tests/core_tests/transaction_tests.cpp b/tests/core_tests/transaction_tests.cpp index d19bdd1f9..efd294336 100644 --- a/tests/core_tests/transaction_tests.cpp +++ b/tests/core_tests/transaction_tests.cpp @@ -72,7 +72,6 @@ bool test_transaction_generation_and_ring_signature() construct_miner_tx(0, 0, 0, 0, 0, miner_acc6.get_keys().m_account_address, tx_mine_6); //fill inputs entry - typedef tx_source_entry::output_entry tx_output_entry; std::vector<tx_source_entry> sources; sources.resize(sources.size()+1); tx_source_entry& src = sources.back(); diff --git a/tests/unit_tests/logging.cpp b/tests/unit_tests/logging.cpp index f11b17412..b3ffb9aa6 100644 --- a/tests/unit_tests/logging.cpp +++ b/tests/unit_tests/logging.cpp @@ -208,3 +208,10 @@ TEST(logging, operator_equals_segfault) el::Logger log2("id2", nullptr); log2 = log1; } + +TEST(logging, empty_configurations_throws) +{ + el::Logger log1("id1", nullptr); + const el::Configurations cfg; + EXPECT_ANY_THROW(log1.configure(cfg)); +} diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index cab600b3d..cc7790c1a 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -303,6 +303,9 @@ TEST(node_server, bind_same_p2p_port) Relevant part about REUSEADDR from man: https://www.man7.org/linux/man-pages/man7/ip.7.html + + For Mac OSX, set the following alias, before running the test, or else it will fail: + sudo ifconfig lo0 alias 127.0.0.2 */ vm.find(nodetool::arg_p2p_bind_ip.name)->second = boost::program_options::variable_value(std::string("127.0.0.2"), false); vm.find(nodetool::arg_p2p_bind_port.name)->second = boost::program_options::variable_value(std::string(port), false); diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 207565e6f..435459f6d 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -53,6 +53,14 @@ class Daemon(object): return self.rpc.send_json_rpc_request(getblocktemplate) get_block_template = getblocktemplate + def get_miner_data(self): + get_miner_data = { + 'method': 'get_miner_data', + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_miner_data) + def add_aux_pow(self, blocktemplate_blob, aux_pow, client = ""): add_aux_pow = { 'method': 'add_aux_pow', |