diff options
137 files changed, 4995 insertions, 832 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aaad0c3a6..8bc1d6498 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,17 +16,19 @@ jobs: build-windows: runs-on: windows-latest + defaults: + run: + shell: msys2 {0} steps: - uses: actions/checkout@v1 with: submodules: recursive - - uses: eine/setup-msys2@v0 + - uses: eine/setup-msys2@v1 with: update: true - - name: install monero dependencies - run: msys2do pacman -S --noconfirm mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git - name: build - run: msys2do make release-static-win64 -j2 + run: make release-static-win64 -j2 build-ubuntu: runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index f8e7c305b..9dacf534f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,3 +15,7 @@ [submodule "external/randomx"] path = external/randomx url = https://github.com/tevador/RandomX +[submodule "external/supercop"] + path = external/supercop + url = https://github.com/monero-project/supercop + branch = monero diff --git a/CMakeLists.txt b/CMakeLists.txt index f63c07a35..2a16e0081 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,6 +210,7 @@ if(NOT MANUAL_SUBMODULES) check_submodule(external/rapidjson) check_submodule(external/trezor-common) check_submodule(external/randomx) + check_submodule(external/supercop) endif() endif() @@ -311,7 +312,7 @@ endif() # elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*") # set(BSDI TRUE) -include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external) +include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external external/supercop/include) if(APPLE) include_directories(SYSTEM /usr/include/malloc) @@ -431,7 +432,7 @@ endif () if (APPLE AND NOT IOS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64 -fvisibility=default -std=c++11") - if (NOT OpenSSL_DIR) + if (NOT OPENSSL_ROOT_DIR) EXECUTE_PROCESS(COMMAND brew --prefix openssl OUTPUT_VARIABLE OPENSSL_ROOT_DIR OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -456,6 +457,9 @@ add_definition_if_function_found(strptime HAVE_STRPTIME) add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP) +set(MONERO_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_include") +include_directories(${MONERO_GENERATED_HEADERS_DIR}) + # Generate header for embedded translations # Generate header for embedded translations, use target toolchain if depends, otherwise use the # lrelease and lupdate binaries from the host @@ -987,6 +991,7 @@ if(SODIUM_LIBRARY) set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}") endif() +include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion add_subdirectory(contrib) add_subdirectory(src) @@ -22,6 +22,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - [Dependencies](#dependencies) - [Internationalization](#Internationalization) - [Using Tor](#using-tor) + - [Pruning](#Pruning) - [Debugging](#Debugging) - [Known issues](#known-issues) @@ -178,10 +179,10 @@ library archives (`.a`). | Graphviz | any | NO | `graphviz` | `graphviz` | `graphviz` | `graphviz` | YES | Documentation | | lrelease | ? | NO | `qttools5-dev-tools` | `qt5-tools` | `qt5-tools` | `qt5-linguist` | YES | Translations | | libhidapi | ? | NO | `libhidapi-dev` | `hidapi` | `hidapi-devel` | `hidapi-devel` | YES | Hardware wallet | -| libusb | ? | NO | `libusb-dev` | `libusb` | `libusb-devel` | `libusb-devel` | YES | Hardware wallet | +| libusb | ? | NO | `libusb-1.0-0-dev` | `libusb` | `libusb-devel` | `libusbx-devel` | YES | Hardware wallet | | libprotobuf | ? | NO | `libprotobuf-dev` | `protobuf` | `protobuf-devel` | `protobuf-devel` | YES | Hardware wallet | | protoc | ? | NO | `protobuf-compiler` | `protobuf` | `protobuf` | `protobuf-compiler` | YES | Hardware wallet | - +| libudev | ? | No | `libudev-dev` | `systemd` | `eudev-libudev-devel` | `systemd-devel` | YES | Hardware wallet | [1] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ``` @@ -189,7 +190,7 @@ build the library binary manually. This can be done with the following command ` Install all dependencies at once on Debian/Ubuntu: -``` sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev doxygen graphviz libpgm-dev qttools5-dev-tools libhidapi-dev libusb-dev libprotobuf-dev protobuf-compiler ``` +``` sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev doxygen graphviz libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev``` Install all dependencies at once on macOS with the provided Brewfile: ``` brew update && brew bundle --file=contrib/brew/Brewfile ``` @@ -468,6 +469,14 @@ c++: error: unable to execute command: Abort trap (core dumped) Then you need to increase the data ulimit size to 2GB and try again: `ulimit -d 2000000` +### On NetBSD: + +Check that the dependencies are present: `pkg_info -c libexecinfo boost-headers boost-libs protobuf readline libusb1 zeromq git-base pkgconf gmake cmake | more`, and install any that are reported missing, using `pkg_add` or from your pkgsrc tree. Readline is optional but worth having. + +Third-party dependencies are usually under `/usr/pkg/`, but if you have a custom setup, adjust the "/usr/pkg" (below) accordingly. + +Clone the monero repository recursively and checkout the most recent release as described above. Then build monero: `gmake BOOST_ROOT=/usr/pkg LDFLAGS="-Wl,-R/usr/pkg/lib" release`. The resulting executables can be found in `build/NetBSD/[Release version]/Release/bin/`. + ### On Solaris: The default Solaris linker can't be used, you have to install GNU ld, then run cmake manually with the path to your copy of GNU ld: @@ -671,6 +680,16 @@ DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip --data-dir /home/amnesia/Persistent/your/directory/to/the/blockchain ``` +## Pruning + +As of May 2020, the full Monero blockchain file is about 80 GB. One can store a pruned blockchain, which is about 28 GB. +A pruned blockchain can only serve part of the historical chain data to other peers, but is otherwise identical in +functionality to the full blockchain. +To use a pruned blockchain, it is best to start the initial sync with --prune-blockchain. However, it is also possible +to prune an existing blockchain using the monero-blockchain-prune tool or using the --prune-blockchain monerod option +with an existing chain. If an existing chain exists, pruning will temporarily require disk space to store both the full +and pruned blockchains. + ## Debugging This section contains general instructions for debugging failed installs or problems encountered with Monero. First, ensure you are running the latest version built from the Github repo. @@ -0,0 +1,61 @@ +# The Current/Future Status of ZMQ in Monero + +## ZMQ Pub/Sub +Client `ZMQ_SUB` sockets must "subscribe" to topics before it receives any data. +This allows filtering on the server side, so network traffic is reduced. Monero +allows for filtering on: (1) format, (2) context, and (3) event. + + * **format** refers to the _wire_ format (i.e. JSON) used to send event + information. + * **context** allows for a reduction in fields for the event, so the + daemon doesn't waste cycles serializing fields that get ignored. + * **event** refers to status changes occurring within the daemon (i.e. new + block to main chain). + + * Formats: + * `json` + * Contexts: + * `full` - the entire block or transaction is transmitted (the hash can be + computed remotely). + * `minimal` - the bare minimum for a remote client to react to an event is + sent. + * Events: + * `chain_main` - changes to the primary/main blockchain. + * `txpool_add` - new _publicly visible_ transactions in the mempool. + Includes previously unseen transactions in a block but _not_ the + `miner_tx`. Does not "re-publish" after a reorg. Includes `do_not_relay` + transactions. + +The subscription topics are formatted as `format-context-event`, with prefix +matching supported by both Monero and ZMQ. The `format`, `context` and `event` +will _never_ have hyphens or colons in their name. For example, subscribing to +`json-minimal-chain_main` will send minimal information in JSON when changes +to the main/primary blockchain occur. Whereas, subscribing to `json-minimal` +will send minimal information in JSON on all available events supported by the +daemon. + +The Monero daemon will ensure that events prefixed by `chain` will be sent in +"chain-order" - the `prev_id` (hash) field will _always_ refer to a previous +block. On rollbacks/reorgs, the event will reference an earlier block in the +chain instead of the last block. The Monero daemon also ensures that +`txpool_add` events are sent before `chain_*` events - the `chain_*` messages +will only serialize miner transactions since the other transactions were +previously published via `txpool_add`. This prevents transactions from being +serialized twice, even when the transaction was first observed in a block. + +ZMQ Pub/Sub will drop messages if the network is congested, so the above rules +for send order are used for detecting lost messages. A missing gap in `height` +or `prev_id` for `chain_*` events indicates a lost pub message. Missing +`txpool_add` messages can only be detected at the next `chain_` message. + +Since blockchain events can be dropped, clients will likely want to have a +timeout against `chain_main` events. The `GetLastBlockHeader` RPC is useful +for checking the current chain state. Dropped messages should be rare in most +conditions. + +The Monero daemon will send a `txpool_add` pub exactly once for each +transaction, even after a reorg or restarts. Clients should use the +`GetTransactionPool` after a reorg to get all transactions that have been put +back into the tx pool or been invalidated due to a double-spend. + + diff --git a/contrib/depends/packages/expat.mk b/contrib/depends/packages/expat.mk index ef81636a2..d73a5e307 100644 --- a/contrib/depends/packages/expat.mk +++ b/contrib/depends/packages/expat.mk @@ -1,6 +1,6 @@ package=expat $(package)_version=2.2.4 -$(package)_download_path=https://downloads.sourceforge.net/project/expat/expat/$($(package)_version) +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_4 $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd372019e diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index 2634423ab..383b88f31 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -1,5 +1,6 @@ # Set the system name to one of Android, Darwin, FreeBSD, Linux, or Windows SET(CMAKE_SYSTEM_NAME @depends@) +SET(CMAKE_SYSTEM_PROCESSOR @arch@) SET(CMAKE_BUILD_TYPE @release_type@) OPTION(STATIC "Link libraries statically" ON) @@ -55,7 +56,7 @@ SET(Boost_NO_SYSTEM_PATHS ON) SET(Boost_USE_STATIC_LIBS ON) SET(Boost_USE_STATIC_RUNTIME ON) -SET(OpenSSL_DIR @prefix@/lib) +SET(OPENSSL_ROOT_DIR @prefix@) SET(ARCHITECTURE @arch@) # for libraries and headers in the target directories @@ -63,14 +64,14 @@ set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) # Find programs on host set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) # Find libs in target set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # Find includes in target -set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR} CACHE STRING "" FORCE) - # specify the cross compiler to be used. Darwin uses clang provided by the SDK. if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") SET(CMAKE_C_COMPILER @prefix@/native/bin/clang) SET(CMAKE_C_COMPILER_TARGET x86_64-apple-darwin11) SET(CMAKE_CXX_COMPILER @prefix@/native/bin/clang++ -stdlib=libc++) SET(CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin11) + SET(CMAKE_ASM_COMPILER_TARGET x86_64-apple-darwin11) + SET(CMAKE_ASM-ATT_COMPILER_TARGET x86_64-apple-darwin11) SET(_CMAKE_TOOLCHAIN_PREFIX x86_64-apple-darwin11-) SET(APPLE True) SET(BUILD_TAG "mac-x64") diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 08d9b8802..219b593b0 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -543,6 +543,31 @@ eof: return it->second.second; } + std::vector<std::string> get_command_list(const std::vector<std::string>& keywords = std::vector<std::string>()) + { + std::vector<std::string> list; + list.reserve(m_command_handlers.size()); + for(auto const& x:m_command_handlers) + { + bool take = true; + for(auto const& y:keywords) + { + bool in_usage = x.second.second.first.find(y) != std::string::npos; + bool in_description = x.second.second.second.find(y) != std::string::npos; + if (!(in_usage || in_description)) + { + take = false; + break; + } + } + if (take) + { + list.push_back(x.first); + } + } + return list; + } + void set_handler(const std::string& cmd, const callback& hndlr, const std::string& usage = "", const std::string& description = "") { lookup::mapped_type & vt = m_command_handlers[cmd]; diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index 5f7202150..a04c63231 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -106,6 +106,14 @@ namespace misc_utils return true; } + template <typename T> + T get_mid(const T &a, const T &b) + { + //returns the average of two numbers; overflow safe and works with at least all integral and floating point types + //(a+b)/2 = (a/2) + (b/2) + ((a - 2*(a/2)) + (b - 2*(b/2)))/2 + return (a/2) + (b/2) + ((a - 2*(a/2)) + (b - 2*(b/2)))/2; + } + template<class type_vec_type> type_vec_type median(std::vector<type_vec_type> &v) { @@ -122,7 +130,7 @@ namespace misc_utils return v[n]; }else {//2, 4, 6... - return (v[n-1] + v[n])/2; + return get_mid<type_vec_type>(v[n-1],v[n]); } } diff --git a/contrib/epee/include/net/abstract_http_client.h b/contrib/epee/include/net/abstract_http_client.h index 787ae2667..1f8bbc605 100644 --- a/contrib/epee/include/net/abstract_http_client.h +++ b/contrib/epee/include/net/abstract_http_client.h @@ -64,6 +64,7 @@ namespace http abstract_http_client() {} virtual ~abstract_http_client() {} bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + virtual bool set_proxy(const std::string& address); virtual void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) = 0; virtual void set_auto_connect(bool auto_connect) = 0; virtual bool connect(std::chrono::milliseconds timeout) = 0; diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 86df48f65..9645e896b 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -885,14 +885,6 @@ namespace net_utils } }; typedef http_simple_client_template<blocked_mode_client> http_simple_client; - - class http_simple_client_factory : public http_client_factory - { - public: - std::unique_ptr<abstract_http_client> create() override { - return std::unique_ptr<epee::net_utils::http::abstract_http_client>(new epee::net_utils::http::http_simple_client()); - } - }; } } } diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 0c0653cd6..ac22cd7a9 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -42,8 +42,17 @@ MINFO("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \ response.m_response_code = 200; \ response.m_response_comment = "Ok"; \ - if(!handle_http_request_map(query_info, response, m_conn_context)) \ - {response.m_response_code = 404;response.m_response_comment = "Not found";} \ + try \ + { \ + if(!handle_http_request_map(query_info, response, m_conn_context)) \ + {response.m_response_code = 404;response.m_response_comment = "Not found";} \ + } \ + catch (const std::exception &e) \ + { \ + MERROR(m_conn_context << "Exception in handle_http_request_map: " << e.what()); \ + response.m_response_code = 500; \ + response.m_response_comment = "Internal Server Error"; \ + } \ return true; \ } @@ -69,9 +78,11 @@ uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ - if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ + bool res = false; \ + try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \ + catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \ + if (!res) \ { \ - MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ @@ -97,9 +108,11 @@ uint64_t ticks1 = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ - if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ + bool res = false; \ + try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \ + catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "()"); } \ + if (!res) \ { \ - MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ @@ -184,7 +197,10 @@ fail_resp.jsonrpc = "2.0"; \ fail_resp.id = req.id; \ MINFO(m_conn_context << "Calling RPC method " << method_name); \ - if(!callback_f(req.params, resp.result, fail_resp.error, &m_conn_context)) \ + bool res = false; \ + try { res = callback_f(req.params, resp.result, fail_resp.error, &m_conn_context); } \ + catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \ + if (!res) \ { \ epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \ return true; \ @@ -203,7 +219,10 @@ fail_resp.jsonrpc = "2.0"; \ fail_resp.id = req.id; \ MINFO(m_conn_context << "calling RPC method " << method_name); \ - if(!callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context)) \ + bool res = false; \ + try { res = callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context); } \ + catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \ + if (!res) \ { \ epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \ return true; \ @@ -217,7 +236,10 @@ { \ PREPARE_OBJECTS_FROM_JSON(command_type) \ MINFO(m_conn_context << "calling RPC method " << method_name); \ - if(!callback_f(req.params, resp.result, &m_conn_context)) \ + bool res = false; \ + try { res = callback_f(req.params, resp.result, &m_conn_context); } \ + catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \ + if (!res) \ { \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index 246cf6ad8..1eeab2dc5 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -28,8 +28,6 @@ #pragma once #include <string> -#include <boost/algorithm/string/predicate.hpp> -#include <boost/asio/ip/address_v6.hpp> #include "int-util.h" // IP addresses are kept in network byte order @@ -40,30 +38,6 @@ namespace epee { namespace net_utils { - - inline - bool is_ipv6_local(const std::string& ip) - { - auto addr = boost::asio::ip::address_v6::from_string(ip); - - // ipv6 link-local unicast addresses are fe80::/10 - bool is_link_local = addr.is_link_local(); - - auto addr_bytes = addr.to_bytes(); - - // ipv6 unique local unicast addresses start with fc00::/7 -- (fcXX or fdXX) - bool is_unique_local_unicast = (addr_bytes[0] == 0xfc || addr_bytes[0] == 0xfd); - - return is_link_local || is_unique_local_unicast; - } - - inline - bool is_ipv6_loopback(const std::string& ip) - { - // ipv6 loopback is ::1 - return boost::asio::ip::address_v6::from_string(ip).is_loopback(); - } - inline bool is_ip_local(uint32_t ip) { diff --git a/contrib/epee/include/rolling_median.h b/contrib/epee/include/rolling_median.h index 11275aa70..088a71d3e 100644 --- a/contrib/epee/include/rolling_median.h +++ b/contrib/epee/include/rolling_median.h @@ -34,6 +34,8 @@ #pragma once +#include "misc_language.h" + #include <stdlib.h> #include <stdint.h> @@ -226,7 +228,7 @@ public: Item v = data[heap[0]]; if (minCt < maxCt) { - v = (v + data[heap[-1]]) / 2; + v = get_mid<Item>(v, data[heap[-1]]); } return v; } diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 4633fa546..cf1262486 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -291,6 +291,7 @@ namespace epee #define BEGIN_INVOKE_MAP2(owner_type) \ template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \ { \ + try { \ typedef owner_type internal_owner_type_name; #define HANDLE_INVOKE2(command_id, func, type_name_in, typename_out) \ @@ -336,7 +337,13 @@ namespace epee LOG_ERROR("Unknown command:" << command); \ on_levin_traffic(context, false, false, true, in_buff.size(), "invalid-command"); \ return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \ + } \ + catch (const std::exception &e) { \ + MERROR("Error in handle_invoke_map: " << e.what()); \ + return LEVIN_ERROR_CONNECTION_TIMEDOUT; /* seems kinda appropriate */ \ + } \ } + } } diff --git a/contrib/epee/src/abstract_http_client.cpp b/contrib/epee/src/abstract_http_client.cpp index 98b5b67d9..540917873 100644 --- a/contrib/epee/src/abstract_http_client.cpp +++ b/contrib/epee/src/abstract_http_client.cpp @@ -137,6 +137,11 @@ namespace http set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options)); return true; } + + bool epee::net_utils::http::abstract_http_client::set_proxy(const std::string& address) + { + return false; + } } } } diff --git a/external/easylogging++/ea_config.h b/external/easylogging++/ea_config.h index 91a671575..4ba0cd611 100644 --- a/external/easylogging++/ea_config.h +++ b/external/easylogging++/ea_config.h @@ -11,7 +11,7 @@ #define ELPP_UTC_DATETIME #ifdef EASYLOGGING_CC -#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__) +#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__ || defined __NetBSD__) #define ELPP_FEATURE_CRASH_LOG #endif #endif diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 0d748c225..bf877c018 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2968,6 +2968,16 @@ void Writer::initializeLogger(Logger *logger, bool needLock) { } void Writer::processDispatch() { + static std::atomic_flag in_dispatch; + if (in_dispatch.test_and_set()) + { + if (m_proceed && m_logger != NULL) + { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + return; + } #if ELPP_LOGGING_ENABLED if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { bool firstDispatched = false; @@ -3006,6 +3016,7 @@ void Writer::processDispatch() { m_logger->releaseLock(); } #endif // ELPP_LOGGING_ENABLED + in_dispatch.clear(); } void Writer::triggerDispatch(void) { diff --git a/external/randomx b/external/randomx -Subproject 7567cef4c6192fb5356bbdd7db802be77be0439 +Subproject 5ce5f4906c1eb166be980f6d83cc80f4112ffc2 diff --git a/external/supercop b/external/supercop new file mode 160000 +Subproject 7d8b6878260061da56ade6d23dc833288659d0a diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 9e977b1b9..a9a7d035f 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -63,6 +63,7 @@ bool matches_category(relay_method method, relay_category category) noexcept { default: case relay_method::local: + case relay_method::forward: case relay_method::stem: return false; case relay_method::block: @@ -79,6 +80,7 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept kept_by_block = 0; do_not_relay = 0; is_local = 0; + is_forwarding = 0; dandelionpp_stem = 0; switch (method) @@ -89,8 +91,8 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept case relay_method::local: is_local = 1; break; - default: - case relay_method::fluff: + case relay_method::forward: + is_forwarding = 1; break; case relay_method::stem: dandelionpp_stem = 1; @@ -98,26 +100,45 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept case relay_method::block: kept_by_block = 1; break; + default: + case relay_method::fluff: + break; } } relay_method txpool_tx_meta_t::get_relay_method() const noexcept { - if (kept_by_block) - return relay_method::block; - if (do_not_relay) - return relay_method::none; - if (is_local) - return relay_method::local; - if (dandelionpp_stem) - return relay_method::stem; + const uint8_t state = + uint8_t(kept_by_block) + + (uint8_t(do_not_relay) << 1) + + (uint8_t(is_local) << 2) + + (uint8_t(is_forwarding) << 3) + + (uint8_t(dandelionpp_stem) << 4); + + switch (state) + { + default: // error case + case 0: + break; + case 1: + return relay_method::block; + case 2: + return relay_method::none; + case 4: + return relay_method::local; + case 8: + return relay_method::forward; + case 16: + return relay_method::stem; + }; return relay_method::fluff; } bool txpool_tx_meta_t::upgrade_relay_method(relay_method method) noexcept { static_assert(relay_method::none < relay_method::local, "bad relay_method value"); - static_assert(relay_method::local < relay_method::stem, "bad relay_method value"); + static_assert(relay_method::local < relay_method::forward, "bad relay_method value"); + static_assert(relay_method::forward < relay_method::stem, "bad relay_method value"); static_assert(relay_method::stem < relay_method::fluff, "bad relay_method value"); static_assert(relay_method::fluff < relay_method::block, "bad relay_method value"); @@ -158,7 +179,7 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) { const transaction &tx = txp.first; @@ -260,12 +281,13 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck time1 = epee::misc_utils::get_tick_count(); uint64_t num_rct_outs = 0; - add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx))); + blobdata miner_bd = tx_to_blob(blk.miner_tx); + add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd))); if (blk.miner_tx.version == 2) num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; - for (const std::pair<transaction, blobdata>& tx : txs) + for (const std::pair<transaction, blobdata_ref>& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index f513651ed..abebb52b4 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -160,7 +160,7 @@ struct txpool_tx_meta_t uint64_t max_used_block_height; uint64_t last_failed_height; uint64_t receive_time; - uint64_t last_relayed_time; //!< If Dandelion++ stem, randomized embargo timestamp. Otherwise, last relayed timestmap. + uint64_t last_relayed_time; //!< If received over i2p/tor, randomized forward time. If Dandelion++stem, randomized embargo time. Otherwise, last relayed timestamp // 112 bytes uint8_t kept_by_block; uint8_t relayed; @@ -169,7 +169,8 @@ struct txpool_tx_meta_t uint8_t pruned: 1; uint8_t is_local: 1; uint8_t dandelionpp_stem : 1; - uint8_t bf_padding: 4; + uint8_t is_forwarding: 1; + uint8_t bf_padding: 3; uint8_t padding[76]; // till 192 bytes @@ -439,7 +440,7 @@ private: * @param tx_prunable_hash the hash of the prunable part of the transaction * @return the transaction ID */ - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; /** * @brief remove data about a transaction @@ -567,7 +568,7 @@ protected: * @param tx_hash_ptr the hash of the transaction, if already calculated * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated */ - void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); + void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric @@ -1523,7 +1524,7 @@ public: * * @param details the details of the transaction to add */ - virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0; + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& details) = 0; /** * @brief update a txpool transaction's metadata @@ -1643,7 +1644,7 @@ public: * @param: data: the metadata for the block * @param: blob: the block's blob */ - virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) = 0; + virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) = 0; /** * @brief get an alternative block by hash @@ -1686,7 +1687,7 @@ public: * * @return false if the function returns false for any transaction, otherwise true */ - virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0; + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0; /** * @brief runs a function over all key images stored @@ -1778,7 +1779,7 @@ public: * * @return false if the function returns false for any output, otherwise true */ - virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const = 0; + virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0; // diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 6ea55d09d..8aa958825 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -856,7 +856,7 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -896,7 +896,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); - const cryptonote::blobdata &blob = txp.second; + const cryptonote::blobdata_ref &blob = txp.second; MDB_val_sized(blobval, blob); unsigned int unprunable_size = tx.unprunable_size; @@ -1756,7 +1756,7 @@ void BlockchainLMDB::unlock() auto_txn.commit(); \ } while(0) -void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta) +void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t &meta) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2308,7 +2308,7 @@ bool BlockchainLMDB::check_pruning() return prune_worker(prune_mode_check, 0); } -bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category category) const +bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob, relay_category category) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2334,8 +2334,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data; if (!meta.matches(category)) continue; - const cryptonote::blobdata *passed_bd = NULL; - cryptonote::blobdata bd; + cryptonote::blobdata_ref bd; if (include_blob) { MDB_val b; @@ -2344,11 +2343,10 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, throw0(DB_ERROR("Failed to find txpool tx blob to match metadata")); if (result) throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str())); - bd.assign(reinterpret_cast<const char*>(b.mv_data), b.mv_size); - passed_bd = &bd; + bd = {reinterpret_cast<const char*>(b.mv_data), b.mv_size}; } - if (!f(txid, meta, passed_bd)) { + if (!f(txid, meta, &bd)) { ret = false; break; } @@ -2359,7 +2357,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, return ret; } -bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, const alt_block_data_t&, const cryptonote::blobdata*)> f, bool include_blob) const +bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, const alt_block_data_t&, const cryptonote::blobdata_ref*)> f, bool include_blob) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2384,15 +2382,13 @@ bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, if (v.mv_size < sizeof(alt_block_data_t)) throw0(DB_ERROR("alt_blocks record is too small")); const alt_block_data_t *data = (const alt_block_data_t*)v.mv_data; - const cryptonote::blobdata *passed_bd = NULL; - cryptonote::blobdata bd; + cryptonote::blobdata_ref bd; if (include_blob) { - bd.assign(reinterpret_cast<const char*>(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t)); - passed_bd = &bd; + bd = {reinterpret_cast<const char*>(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t)}; } - if (!f(blkid, *data, passed_bd)) { + if (!f(blkid, *data, &bd)) { ret = false; break; } @@ -3604,8 +3600,7 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st if (ret) throw0(DB_ERROR("Failed to enumerate blocks")); uint64_t height = *(const uint64_t*)k.mv_data; - blobdata bd; - bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size}; block b; if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); @@ -3660,15 +3655,16 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& if (ret) throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); transaction tx; - blobdata bd; - bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); if (pruned) { + blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size}; if (!parse_and_validate_tx_base_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); } else { + blobdata bd; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); ret = mdb_cursor_get(m_cur_txs_prunable, &k, &v, MDB_SET); if (ret) throw0(DB_ERROR(lmdb_error("Failed to get prunable tx data the db: ", ret).c_str())); @@ -4402,7 +4398,7 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const return ret; } -void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) +void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -4969,7 +4965,7 @@ void BlockchainLMDB::migrate_0_1() } MDB_dbi o_txs; - blobdata bd; + blobdata_ref bd; block b; MDB_val hk; @@ -5051,7 +5047,7 @@ void BlockchainLMDB::migrate_0_1() } else if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str())); - bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + bd = {reinterpret_cast<char*>(v.mv_data), v.mv_size}; if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); @@ -5062,7 +5058,7 @@ void BlockchainLMDB::migrate_0_1() result = mdb_cursor_get(c_txs, &hk, &v, MDB_SET); if (result) throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); - bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + bd = {reinterpret_cast<char*>(v.mv_data), v.mv_size}; if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]); @@ -5184,8 +5180,7 @@ void BlockchainLMDB::migrate_1_2() else if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); - cryptonote::blobdata bd; - bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + cryptonote::blobdata bd{reinterpret_cast<char*>(v.mv_data), v.mv_size}; transaction tx; if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 5abb8014f..568882ae5 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -283,7 +283,7 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; - virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta); + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& meta); virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta); virtual uint64_t get_txpool_tx_count(relay_category category = relay_category::broadcasted) const; virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const; @@ -296,20 +296,20 @@ public: virtual bool update_pruning(); virtual bool check_pruning(); - virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob); + virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob); virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob); virtual void remove_alt_block(const crypto::hash &blkid); virtual uint64_t get_alt_block_count(); virtual void drop_alt_blocks(); - virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const; + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const; virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const; virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const; virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; - virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const; + virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const; virtual uint64_t add_block( const std::pair<block, blobdata>& blk , size_t block_weight @@ -376,7 +376,7 @@ private: virtual void remove_block(); - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 92911d081..9e74b33f1 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -111,7 +111,7 @@ public: virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const override { return std::vector<std::vector<uint64_t>>(); } virtual bool has_key_image(const crypto::key_image& img) const override { return false; } virtual void remove_block() override { } - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;} + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;} virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {} virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;} virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {} @@ -127,7 +127,7 @@ public: virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const override { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); } virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const override { return false; } - virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) override {} + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const cryptonote::txpool_tx_meta_t& details) override {} virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {} virtual uint64_t get_txpool_tx_count(relay_category tx_relay = relay_category::broadcasted) const override { return 0; } virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const override { return false; } @@ -136,7 +136,7 @@ public: virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const override { return false; } virtual uint64_t get_database_size() const override { return 0; } virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const override { return ""; } - virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; } + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; } virtual void add_block( const cryptonote::block& blk , size_t block_weight @@ -160,12 +160,12 @@ public: virtual uint64_t get_max_block_size() override { return 100000000; } virtual void add_max_block_size(uint64_t sz) override { } - virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) override {} + virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) override {} virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob) override { return false; } virtual void remove_alt_block(const crypto::hash &blkid) override {} virtual uint64_t get_alt_block_count() override { return 0; } virtual void drop_alt_blocks() override {} - virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const override { return true; } + virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const override { return true; } }; } diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index a847f6a9d..a8197483f 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -28,8 +28,6 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> -#include <boost/archive/portable_binary_iarchive.hpp> -#include <boost/archive/portable_binary_oarchive.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index c9c815f2a..1f728b4e5 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -68,6 +68,9 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<bool> arg_outputs = {"with-outputs", "with output stats", false}; const command_line::arg_descriptor<bool> arg_ringsize = {"with-ringsize", "with ringsize stats", false}; const command_line::arg_descriptor<bool> arg_hours = {"with-hours", "with txns per hour", false}; + const command_line::arg_descriptor<bool> arg_emission = {"with-emission", "with coin emission", false}; + const command_line::arg_descriptor<bool> arg_fees = {"with-fees", "with txn fees", false}; + const command_line::arg_descriptor<bool> arg_diff = {"with-diff", "with difficulty", false}; command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); @@ -79,6 +82,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_outputs); command_line::add_arg(desc_cmd_sett, arg_ringsize); command_line::add_arg(desc_cmd_sett, arg_hours); + command_line::add_arg(desc_cmd_sett, arg_emission); + command_line::add_arg(desc_cmd_sett, arg_fees); + command_line::add_arg(desc_cmd_sett, arg_diff); command_line::add_arg(desc_cmd_only, command_line::arg_help); po::options_description desc_options("Allowed options"); @@ -120,6 +126,9 @@ int main(int argc, char* argv[]) bool do_outputs = command_line::get_arg(vm, arg_outputs); bool do_ringsize = command_line::get_arg(vm, arg_ringsize); bool do_hours = command_line::get_arg(vm, arg_hours); + bool do_emission = command_line::get_arg(vm, arg_emission); + bool do_fees = command_line::get_arg(vm, arg_fees); + bool do_diff = command_line::get_arg(vm, arg_diff); LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); std::unique_ptr<Blockchain> core_storage; @@ -177,12 +186,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' // spit out a comment that GnuPlot can use as an index std::cout << ENDL << "# DATA" << ENDL; std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes"; + if (do_emission) + std::cout << "\tEmission/day\tEmission"; + if (do_fees) + std::cout << "\tFees/day\tFees"; + if (do_diff) + std::cout << "\tDiffMin\tDiffMax\tDiffAvg"; if (do_inputs) std::cout << "\tInMin\tInMax\tInAvg"; if (do_outputs) std::cout << "\tOutMin\tOutMax\tOutAvg"; if (do_ringsize) std::cout << "\tRingMin\tRingMax\tRingAvg"; + if (do_inputs || do_outputs || do_ringsize) + std::cout << std::setprecision(2) << std::fixed; if (do_hours) { char buf[8]; unsigned int i; @@ -193,14 +210,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' } std::cout << ENDL; +#define MAX_INOUT 0xffffffff +#define MAX_RINGS 0xffffffff + struct tm prevtm = {0}, currtm; uint64_t prevsz = 0, currsz = 0; uint64_t prevtxs = 0, currtxs = 0; uint64_t currblks = 0; uint64_t totins = 0, totouts = 0, totrings = 0; - uint32_t minins = 10, maxins = 0; - uint32_t minouts = 10, maxouts = 0; - uint32_t minrings = 50, maxrings = 0; + boost::multiprecision::uint128_t prevemission = 0, prevfees = 0; + boost::multiprecision::uint128_t emission = 0, fees = 0; + boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0; + uint32_t minins = MAX_INOUT, maxins = 0; + uint32_t minouts = MAX_INOUT, maxouts = 0; + uint32_t minrings = MAX_RINGS, maxrings = 0; uint32_t io, tottxs = 0; uint32_t txhr[24] = {0}; unsigned int i; @@ -230,34 +253,50 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; prevsz += currsz; currsz = 0; - currblks = 0; prevtxs += currtxs; currtxs = 0; if (!tottxs) tottxs = 1; + if (do_emission) { + std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission); + prevemission += emission; + emission = 0; + } + if (do_fees) { + std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees); + prevfees += fees; + fees = 0; + } + if (do_diff) { + std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks; + mindiff = 0; maxdiff = 0; totdiff = 0; + } if (do_inputs) { - std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs; - minins = 10; maxins = 0; totins = 0; + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs; + minins = MAX_INOUT; maxins = 0; totins = 0; } if (do_outputs) { - std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs; - minouts = 10; maxouts = 0; totouts = 0; + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs; + minouts = MAX_INOUT; maxouts = 0; totouts = 0; } if (do_ringsize) { - std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs; - minrings = 50; maxrings = 0; totrings = 0; + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs; + minrings = MAX_RINGS; maxrings = 0; totrings = 0; } - tottxs = 0; if (do_hours) { for (i=0; i<24; i++) { std::cout << "\t" << txhr[i]; txhr[i] = 0; } } + currblks = 0; + tottxs = 0; std::cout << ENDL; } skip: currsz += bd.size(); + uint64_t coinbase_amount; + uint64_t tx_fee_amount = 0; for (const auto& tx_id : blk.tx_hashes) { if (tx_id == crypto::null_hash) @@ -275,7 +314,12 @@ skip: return 1; } currsz += bd.size(); + if (db->get_prunable_tx_blob(tx_id, bd)) + currsz += bd.size(); currtxs++; + if (do_fees || do_emission) { + tx_fee_amount += get_tx_fee(tx); + } if (do_hours) txhr[currtm.tm_hour]++; if (do_inputs) { @@ -306,6 +350,21 @@ skip: } tottxs++; } + if (do_diff) { + difficulty_type diff = db->get_block_difficulty(h); + if (!mindiff || diff < mindiff) + mindiff = diff; + if (diff > maxdiff) + maxdiff = diff; + totdiff += diff; + } + if (do_emission) { + coinbase_amount = get_outs_money_amount(blk.miner_tx); + emission += coinbase_amount - tx_fee_amount; + } + if (do_fees) { + fees += tx_fee_amount; + } currblks++; if (stop_requested) diff --git a/src/common/notify.cpp b/src/common/notify.cpp index e2df5096d..f31100214 100644 --- a/src/common/notify.cpp +++ b/src/common/notify.cpp @@ -62,7 +62,7 @@ static void replace(std::vector<std::string> &v, const char *tag, const char *s) boost::replace_all(str, tag, s); } -int Notify::notify(const char *tag, const char *s, ...) +int Notify::notify(const char *tag, const char *s, ...) const { std::vector<std::string> margs = args; diff --git a/src/common/notify.h b/src/common/notify.h index f813e8def..65d4e1072 100644 --- a/src/common/notify.h +++ b/src/common/notify.h @@ -38,8 +38,12 @@ class Notify { public: Notify(const char *spec); + Notify(const Notify&) = default; + Notify(Notify&&) = default; + Notify& operator=(const Notify&) = default; + Notify& operator=(Notify&&) = default; - int notify(const char *tag, const char *s, ...); + int notify(const char *tag, const char *s, ...) const; private: std::string filename; @@ -47,3 +51,4 @@ private: }; } + diff --git a/src/common/updates.cpp b/src/common/updates.cpp index c51c4320f..61a76f5da 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -102,6 +102,8 @@ namespace tools const char *base = user ? "https://downloads.getmonero.org/" : "https://updates.getmonero.org/"; #ifdef _WIN32 static const char *extension = strncmp(buildtag.c_str(), "source", 6) ? (strncmp(buildtag.c_str(), "install-", 8) ? ".zip" : ".exe") : ".tar.bz2"; +#elif defined(__APPLE__) + static const char *extension = strncmp(software.c_str(), "monero-gui", 10) ? ".tar.bz2" : ".dmg"; #else static const char extension[] = ".tar.bz2"; #endif diff --git a/src/common/util.cpp b/src/common/util.cpp index 138ac4294..433cb4919 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1116,7 +1116,7 @@ std::string get_nix_version_display_string() static constexpr const byte_map sizes[] = { {"%.0f B", 1024}, - {"%.2f KB", 1024 * 1024}, + {"%.2f kB", 1024 * 1024}, {"%.2f MB", std::uint64_t(1024) * 1024 * 1024}, {"%.2f GB", std::uint64_t(1024) * 1024 * 1024 * 1024}, {"%.2f TB", std::uint64_t(1024) * 1024 * 1024 * 1024 * 1024} diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 318e6dc57..3b33fe90a 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -116,3 +116,6 @@ endif() # cheat because cmake and ccache hate each other set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C) + +# Must be done last, because it references libraries in this directory +add_subdirectory(wallet) diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 1e4a6d33f..4cfe83d54 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -43,6 +43,8 @@ #include "crypto.h" #include "hash.h" +#include "cryptonote_config.h" + namespace { static void local_abort(const char *msg) { @@ -261,11 +263,24 @@ namespace crypto { ec_point comm; }; + // Used in v1 tx proofs + struct s_comm_2_v1 { + hash msg; + ec_point D; + ec_point X; + ec_point Y; + }; + + // Used in v1/v2 tx proofs struct s_comm_2 { hash msg; ec_point D; ec_point X; ec_point Y; + hash sep; // domain separation + ec_point R; + ec_point A; + ec_point B; }; void crypto_ops::generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) { @@ -321,6 +336,86 @@ namespace crypto { return sc_isnonzero(&c) == 0; } + // Generate a proof of knowledge of `r` such that (`R = rG` and `D = rA`) or (`R = rB` and `D = rA`) via a Schnorr proof + // This handles use cases for both standard addresses and subaddresses + // + // NOTE: This generates old v1 proofs, and is for TESTING ONLY + void crypto_ops::generate_tx_proof_v1(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) { + // sanity check + ge_p3 R_p3; + ge_p3 A_p3; + ge_p3 B_p3; + ge_p3 D_p3; + if (ge_frombytes_vartime(&R_p3, &R) != 0) throw std::runtime_error("tx pubkey is invalid"); + if (ge_frombytes_vartime(&A_p3, &A) != 0) throw std::runtime_error("recipient view pubkey is invalid"); + if (B && ge_frombytes_vartime(&B_p3, &*B) != 0) throw std::runtime_error("recipient spend pubkey is invalid"); + if (ge_frombytes_vartime(&D_p3, &D) != 0) throw std::runtime_error("key derivation is invalid"); +#if !defined(NDEBUG) + { + assert(sc_check(&r) == 0); + // check R == r*G or R == r*B + public_key dbg_R; + if (B) + { + ge_p2 dbg_R_p2; + ge_scalarmult(&dbg_R_p2, &r, &B_p3); + ge_tobytes(&dbg_R, &dbg_R_p2); + } + else + { + ge_p3 dbg_R_p3; + ge_scalarmult_base(&dbg_R_p3, &r); + ge_p3_tobytes(&dbg_R, &dbg_R_p3); + } + assert(R == dbg_R); + // check D == r*A + ge_p2 dbg_D_p2; + ge_scalarmult(&dbg_D_p2, &r, &A_p3); + public_key dbg_D; + ge_tobytes(&dbg_D, &dbg_D_p2); + assert(D == dbg_D); + } +#endif + + // pick random k + ec_scalar k; + random_scalar(k); + + s_comm_2_v1 buf; + buf.msg = prefix_hash; + buf.D = D; + + if (B) + { + // compute X = k*B + ge_p2 X_p2; + ge_scalarmult(&X_p2, &k, &B_p3); + ge_tobytes(&buf.X, &X_p2); + } + else + { + // compute X = k*G + ge_p3 X_p3; + ge_scalarmult_base(&X_p3, &k); + ge_p3_tobytes(&buf.X, &X_p3); + } + + // compute Y = k*A + ge_p2 Y_p2; + ge_scalarmult(&Y_p2, &k, &A_p3); + ge_tobytes(&buf.Y, &Y_p2); + + // sig.c = Hs(Msg || D || X || Y) + hash_to_scalar(&buf, sizeof(buf), sig.c); + + // sig.r = k - sig.c*r + sc_mulsub(&sig.r, &sig.c, &unwrap(r), &k); + } + + // Generate a proof of knowledge of `r` such that (`R = rG` and `D = rA`) or (`R = rB` and `D = rA`) via a Schnorr proof + // This handles use cases for both standard addresses and subaddresses + // + // Generates only proofs for InProofV2 and OutProofV2 void crypto_ops::generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) { // sanity check ge_p3 R_p3; @@ -362,10 +457,20 @@ namespace crypto { ec_scalar k; random_scalar(k); + // if B is not present + static const ec_point zero = {{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }}; + s_comm_2 buf; buf.msg = prefix_hash; buf.D = D; - + buf.R = R; + buf.A = A; + if (B) + buf.B = *B; + else + buf.B = zero; + cn_fast_hash(config::HASH_KEY_TXPROOF_V2, sizeof(config::HASH_KEY_TXPROOF_V2)-1, buf.sep); + if (B) { // compute X = k*B @@ -386,7 +491,7 @@ namespace crypto { ge_scalarmult(&Y_p2, &k, &A_p3); ge_tobytes(&buf.Y, &Y_p2); - // sig.c = Hs(Msg || D || X || Y) + // sig.c = Hs(Msg || D || X || Y || sep || R || A || B) hash_to_scalar(&buf, sizeof(buf), sig.c); // sig.r = k - sig.c*r @@ -395,7 +500,8 @@ namespace crypto { memwipe(&k, sizeof(k)); } - bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) { + // Verify a proof: either v1 (version == 1) or v2 (version == 2) + bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig, const int version) { // sanity check ge_p3 R_p3; ge_p3 A_p3; @@ -467,14 +573,31 @@ namespace crypto { ge_p2 Y_p2; ge_p1p1_to_p2(&Y_p2, &Y_p1p1); - // compute c2 = Hs(Msg || D || X || Y) + // Compute hash challenge + // for v1, c2 = Hs(Msg || D || X || Y) + // for v2, c2 = Hs(Msg || D || X || Y || sep || R || A || B) + + // if B is not present + static const ec_point zero = {{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }}; + s_comm_2 buf; buf.msg = prefix_hash; buf.D = D; + buf.R = R; + buf.A = A; + if (B) + buf.B = *B; + else + buf.B = zero; + cn_fast_hash(config::HASH_KEY_TXPROOF_V2, sizeof(config::HASH_KEY_TXPROOF_V2)-1, buf.sep); ge_tobytes(&buf.X, &X_p2); ge_tobytes(&buf.Y, &Y_p2); ec_scalar c2; - hash_to_scalar(&buf, sizeof(s_comm_2), c2); + + // Hash depends on version + if (version == 1) hash_to_scalar(&buf, sizeof(s_comm_2) - 3*sizeof(ec_point) - sizeof(hash), c2); + else if (version == 2) hash_to_scalar(&buf, sizeof(s_comm_2), c2); + else return false; // test if c2 == sig.c sc_sub(&c2, &c2, &sig.c); diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 70d463a16..7ddc0150f 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -132,8 +132,10 @@ namespace crypto { friend bool check_signature(const hash &, const public_key &, const signature &); static void generate_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &); friend void generate_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &); - static bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &); - friend bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &); + static void generate_tx_proof_v1(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &); + friend void generate_tx_proof_v1(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &); + static bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &, const int); + friend bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &, const int); static void generate_key_image(const public_key &, const secret_key &, key_image &); friend void generate_key_image(const public_key &, const secret_key &, key_image &); static void generate_ring_signature(const hash &, const key_image &, @@ -248,8 +250,11 @@ namespace crypto { inline void generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) { crypto_ops::generate_tx_proof(prefix_hash, R, A, B, D, r, sig); } - inline bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) { - return crypto_ops::check_tx_proof(prefix_hash, R, A, B, D, sig); + inline void generate_tx_proof_v1(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) { + crypto_ops::generate_tx_proof_v1(prefix_hash, R, A, B, D, r, sig); + } + inline bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig, const int version) { + return crypto_ops::check_tx_proof(prefix_hash, R, A, B, D, sig, version); } /* To send money to a key: diff --git a/src/crypto/wallet/CMakeLists.txt b/src/crypto/wallet/CMakeLists.txt new file mode 100644 index 000000000..4ed986dce --- /dev/null +++ b/src/crypto/wallet/CMakeLists.txt @@ -0,0 +1,62 @@ +# Copyright (c) 2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# Possibly user defined values. +# +set(MONERO_WALLET_CRYPTO_LIBRARY "auto" CACHE STRING "Select a wallet crypto library") + +# +# If the user specified "auto", detect best library defaulting to internal. +# +if (${MONERO_WALLET_CRYPTO_LIBRARY} STREQUAL "auto") + monero_crypto_autodetect(AVAILABLE BEST) + if (DEFINED BEST) + message("Wallet crypto is using ${BEST} backend") + set(MONERO_WALLET_CRYPTO_LIBRARY ${BEST}) + else () + message("Defaulting to internal crypto library for wallet") + set(MONERO_WALLET_CRYPTO_LIBRARY "cn") + endif () +endif () + +# +# Configure library target "wallet-crypto" - clients will use this as a +# library dependency which in turn will depend on the crypto library selected. +# +if (${MONERO_WALLET_CRYPTO_LIBRARY} STREQUAL "cn") + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/empty.h.in ${MONERO_GENERATED_HEADERS_DIR}/crypto/wallet/ops.h) + add_library(wallet-crypto ALIAS cncrypto) +else () + monero_crypto_generate_header(${MONERO_WALLET_CRYPTO_LIBRARY} "${MONERO_GENERATED_HEADERS_DIR}/crypto/wallet/ops.h") + monero_crypto_get_target(${MONERO_WALLET_CRYPTO_LIBRARY} CRYPTO_TARGET) + add_library(wallet-crypto $<TARGET_OBJECTS:${CRYPTO_TARGET}>) + target_link_libraries(wallet-crypto cncrypto) +endif () + + diff --git a/src/serialization/vector.h b/src/crypto/wallet/crypto.h index 1fca1d485..a4c5d5a07 100644 --- a/src/serialization/vector.h +++ b/src/crypto/wallet/crypto.h @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2020, The Monero Project -// +// Copyright (c) 2020, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL @@ -25,41 +25,32 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once -#include <vector> -#include "serialization.h" - -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::vector<T> &v); -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::vector<T> &v); - -namespace serialization -{ - namespace detail - { - template <typename T> - void do_reserve(std::vector<T> &c, size_t N) - { - c.reserve(N); - } - - template <typename T> - void do_add(std::vector<T> &c, T &&e) - { - c.emplace_back(std::move(e)); - } +#include <cstddef> +#include "crypto/wallet/ops.h" + +namespace crypto { + namespace wallet { +// if C functions defined from external/supercop - cmake generates crypto/wallet/ops.h +#if defined(monero_crypto_generate_key_derivation) + inline + bool generate_key_derivation(const public_key &tx_pub, const secret_key &view_sec, key_derivation &out) + { + return monero_crypto_generate_key_derivation(out.data, tx_pub.data, view_sec.data) == 0; + } + + inline + bool derive_subaddress_public_key(const public_key &output_pub, const key_derivation &d, std::size_t index, public_key &out) + { + ec_scalar scalar; + derivation_to_scalar(d, index, scalar); + return monero_crypto_generate_subaddress_public_key(out.data, output_pub.data, scalar.data) == 0; + } +#else + using ::crypto::generate_key_derivation; + using ::crypto::derive_subaddress_public_key; +#endif } } - -#include "container.h" - -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); } - diff --git a/src/crypto/wallet/empty.h.in b/src/crypto/wallet/empty.h.in new file mode 100644 index 000000000..ac252e1bd --- /dev/null +++ b/src/crypto/wallet/empty.h.in @@ -0,0 +1,31 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// Left empty so internal cryptonote crypto library is used. diff --git a/src/cryptonote_basic/blobdatatype.h b/src/cryptonote_basic/blobdatatype.h index 6906e0c9d..7f899993b 100644 --- a/src/cryptonote_basic/blobdatatype.h +++ b/src/cryptonote_basic/blobdatatype.h @@ -31,10 +31,11 @@ #pragma once #include <string> +#include <boost/utility/string_ref_fwd.hpp> #include "span.h" namespace cryptonote { typedef std::string blobdata; - typedef epee::span<const char> blobdata_ref; + typedef boost::string_ref blobdata_ref; } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index bc6a378f2..c70ae1df1 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -37,7 +37,7 @@ #include <sstream> #include <atomic> #include "serialization/variant.h" -#include "serialization/vector.h" +#include "serialization/containers.h" #include "serialization/binary_archive.h" #include "serialization/json_archive.h" #include "serialization/debug_archive.h" diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 541393fa9..2600854a9 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -34,7 +34,6 @@ using namespace epee; #include "cryptonote_basic_impl.h" #include "string_tools.h" #include "serialization/binary_utils.h" -#include "serialization/container.h" #include "cryptonote_format_utils.h" #include "cryptonote_config.h" #include "misc_language.h" diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 6f89cc7ae..b3d39a616 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -36,7 +36,6 @@ #include <boost/serialization/set.hpp> #include <boost/serialization/map.hpp> #include <boost/serialization/is_bitwise_serializable.hpp> -#include <boost/archive/binary_iarchive.hpp> #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> #include "cryptonote_basic.h" diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 3fd059ac1..d808a9c1d 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -209,7 +209,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx) + bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx) { std::stringstream ss; ss << tx_blob; @@ -222,7 +222,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx) + bool parse_and_validate_tx_base_from_blob(const blobdata_ref& tx_blob, transaction& tx) { std::stringstream ss; ss << tx_blob; @@ -234,7 +234,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx) + bool parse_and_validate_tx_prefix_from_blob(const blobdata_ref& tx_blob, transaction_prefix& tx) { std::stringstream ss; ss << tx_blob; @@ -244,7 +244,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash) + bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash) { std::stringstream ss; ss << tx_blob; @@ -258,7 +258,7 @@ namespace cryptonote return get_transaction_hash(tx, tx_hash); } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) + bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) { if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) return false; @@ -958,7 +958,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - void get_blob_hash(const epee::span<const char>& blob, crypto::hash& res) + void get_blob_hash(const blobdata_ref& blob, crypto::hash& res) { cn_fast_hash(blob.data(), blob.size(), res); } @@ -1045,7 +1045,7 @@ namespace cryptonote return h; } //--------------------------------------------------------------- - crypto::hash get_blob_hash(const epee::span<const char>& blob) + crypto::hash get_blob_hash(const blobdata_ref& blob) { crypto::hash h = null_hash; get_blob_hash(blob, h); @@ -1065,7 +1065,7 @@ namespace cryptonote return get_transaction_hash(t, res, NULL); } //--------------------------------------------------------------- - bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res) + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blob, crypto::hash& res) { if (t.version == 1) return false; @@ -1073,7 +1073,7 @@ namespace cryptonote if (blob && unprunable_size) { CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); - cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + cryptonote::get_blob_hash(blobdata_ref(blob->data() + unprunable_size, blob->size() - unprunable_size), res); } else { @@ -1090,7 +1090,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata) + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blobdata) { crypto::hash res; if (t.is_prunable_hash_valid()) @@ -1168,7 +1168,7 @@ namespace cryptonote // base rct CHECK_AND_ASSERT_MES(prefix_size <= unprunable_size && unprunable_size <= blob.size(), false, "Inconsistent transaction prefix, unprunable and blob sizes"); - cryptonote::get_blob_hash(epee::span<const char>(blob.data() + prefix_size, unprunable_size - prefix_size), hashes[1]); + cryptonote::get_blob_hash(blobdata_ref(blob.data() + prefix_size, unprunable_size - prefix_size), hashes[1]); // prunable rct if (t.rct_signatures.type == rct::RCTTypeNull) @@ -1177,7 +1177,8 @@ namespace cryptonote } else { - CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, &blob, hashes[2]), false, "Failed to get tx prunable hash"); + cryptonote::blobdata_ref blobref(blob); + CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, &blobref, hashes[2]), false, "Failed to get tx prunable hash"); } // the tx hash is the hash of the 3 hashes @@ -1241,13 +1242,15 @@ namespace cryptonote return blob; } //--------------------------------------------------------------- - bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob) + bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata_ref *blob) { blobdata bd; + blobdata_ref bdref; if (!blob) { bd = block_to_blob(b); - blob = &bd; + bdref = bd; + blob = &bdref; } bool hash_result = get_object_hash(get_block_hashing_blob(b), res); @@ -1330,7 +1333,7 @@ namespace cryptonote return res; } //--------------------------------------------------------------- - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash) + bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b, crypto::hash *block_hash) { std::stringstream ss; ss << b_blob; @@ -1348,12 +1351,12 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b) + bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b) { return parse_and_validate_block_from_blob(b_blob, b, NULL); } //--------------------------------------------------------------- - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash) + bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b, crypto::hash &block_hash) { return parse_and_validate_block_from_blob(b_blob, b, &block_hash); } diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 5639e38d0..636a88b9a 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -52,11 +52,11 @@ namespace cryptonote crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx, hw::device &hwdev); void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); - bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx); - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash); - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); - bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx); + bool parse_and_validate_tx_prefix_from_blob(const blobdata_ref& tx_blob, transaction_prefix& tx); + bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); + bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash); + bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx); + bool parse_and_validate_tx_base_from_blob(const blobdata_ref& tx_blob, transaction& tx); bool is_v1_tx(const blobdata_ref& tx_blob); bool is_v1_tx(const blobdata& tx_blob); @@ -102,27 +102,27 @@ namespace cryptonote bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); void get_blob_hash(const blobdata& blob, crypto::hash& res); - void get_blob_hash(const epee::span<const char>& blob, crypto::hash& res); + void get_blob_hash(const blobdata_ref& blob, crypto::hash& res); crypto::hash get_blob_hash(const blobdata& blob); - crypto::hash get_blob_hash(const epee::span<const char>& blob); + crypto::hash get_blob_hash(const blobdata_ref& blob); std::string short_hash_str(const crypto::hash& h); crypto::hash get_transaction_hash(const transaction& t); bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); - bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res); - crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob = NULL); + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blob, crypto::hash& res); + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blob = NULL); bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash); blobdata get_block_hashing_blob(const block& b); - bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob = NULL); + bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata_ref *blob = NULL); bool get_block_hash(const block& b, crypto::hash& res); crypto::hash get_block_hash(const block& b); - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash); - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b); - bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash); + bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b, crypto::hash *block_hash); + bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b); + bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b, crypto::hash &block_hash); bool get_inputs_money_amount(const transaction& tx, uint64_t& money); uint64_t get_outs_money_amount(const transaction& tx); bool check_inputs_types_supported(const transaction& tx); diff --git a/src/serialization/deque.h b/src/cryptonote_basic/events.h index b029ed430..6c6742215 100644 --- a/src/serialization/deque.h +++ b/src/cryptonote_basic/events.h @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2020, The Monero Project -// +// Copyright (c) 2020, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL @@ -25,40 +25,22 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once -#include <deque> +#include "crypto/hash.h" +#include "cryptonote_basic/cryptonote_basic.h" -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::deque<T> &v); -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::deque<T> &v); - -namespace serialization +namespace cryptonote { - namespace detail + /*! Transactions are expensive to move or copy (lots of 32-byte internal + buffers). This allows `cryptonote::core` to do a single notification for + a vector of transactions, without having to move/copy duplicate or invalid + transactions. */ + struct txpool_event { - template <typename T> - void do_reserve(std::deque<T> &c, size_t N) - { - c.reserve(N); - } - - template <typename T> - void do_add(std::deque<T> &c, T &&e) - { - c.emplace_back(std::move(e)); - } - } + cryptonote::transaction tx; + crypto::hash hash; + bool res; //!< Listeners must ignore `tx` when this is false. + }; } - -#include "serialization.h" - -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); } - diff --git a/src/cryptonote_basic/fwd.h b/src/cryptonote_basic/fwd.h new file mode 100644 index 000000000..d54223461 --- /dev/null +++ b/src/cryptonote_basic/fwd.h @@ -0,0 +1,36 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace cryptonote +{ + struct block; + class transaction; + struct txpool_event; +} diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 3cbc4e5a4..b23253d4a 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -91,17 +91,16 @@ namespace cryptonote uint64_t get_block_reward() const { return m_block_reward; } static constexpr uint8_t BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE = 90; - static constexpr uint8_t BACKGROUND_MINING_MIN_IDLE_THRESHOLD_PERCENTAGE = 50; + static constexpr uint8_t BACKGROUND_MINING_MIN_IDLE_THRESHOLD_PERCENTAGE = 0; static constexpr uint8_t BACKGROUND_MINING_MAX_IDLE_THRESHOLD_PERCENTAGE = 99; static constexpr uint16_t BACKGROUND_MINING_DEFAULT_MIN_IDLE_INTERVAL_IN_SECONDS = 10; static constexpr uint16_t BACKGROUND_MINING_MIN_MIN_IDLE_INTERVAL_IN_SECONDS = 10; static constexpr uint16_t BACKGROUND_MINING_MAX_MIN_IDLE_INTERVAL_IN_SECONDS = 3600; static constexpr uint8_t BACKGROUND_MINING_DEFAULT_MINING_TARGET_PERCENTAGE = 40; - static constexpr uint8_t BACKGROUND_MINING_MIN_MINING_TARGET_PERCENTAGE = 5; - static constexpr uint8_t BACKGROUND_MINING_MAX_MINING_TARGET_PERCENTAGE = 50; + static constexpr uint8_t BACKGROUND_MINING_MIN_MINING_TARGET_PERCENTAGE = 1; + static constexpr uint8_t BACKGROUND_MINING_MAX_MINING_TARGET_PERCENTAGE = 100; static constexpr uint8_t BACKGROUND_MINING_MINER_MONITOR_INVERVAL_IN_SECONDS = 10; static constexpr uint64_t BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS = 400; // ramp up - static constexpr uint64_t BACKGROUND_MINING_MIN_MINER_EXTRA_SLEEP_MILLIS = 5; private: bool worker_thread(); diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 51912ce58..8051ee9fa 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -117,6 +117,11 @@ #define CRYPTONOTE_NOISE_BYTES 3*1024 // 3 KiB #define CRYPTONOTE_NOISE_CHANNELS 2 // Max outgoing connections per zone used for noise/covert sending +// Both below are in seconds. The idea is to delay forwarding from i2p/tor +// to ipv4/6, such that 2+ incoming connections _could_ have sent the tx +#define CRYPTONOTE_FORWARD_DELAY_BASE (CRYPTONOTE_NOISE_MIN_DELAY + CRYPTONOTE_NOISE_DELAY_RANGE) +#define CRYPTONOTE_FORWARD_DELAY_AVERAGE (CRYPTONOTE_FORWARD_DELAY_BASE + (CRYPTONOTE_FORWARD_DELAY_BASE / 2)) + #define CRYPTONOTE_MAX_FRAGMENTS 20 // ~20 * NOISE_BYTES max payload size for covert/noise send #define COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT 1000 @@ -220,6 +225,7 @@ namespace config const unsigned char HASH_KEY_RPC_PAYMENT_NONCE = 0x58; const unsigned char HASH_KEY_MEMORY = 'k'; const unsigned char HASH_KEY_MULTISIG[] = {'M', 'u', 'l', 't' , 'i', 's', 'i', 'g', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + const unsigned char HASH_KEY_TXPROOF_V2[] = "TXPROOF_V2"; namespace testnet { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index dafc44803..20dc7f9fb 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1234,10 +1234,15 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); - std::shared_ptr<tools::Notify> block_notify = m_block_notify; - if (block_notify) - for (const auto &bei: alt_chain) - block_notify->notify("%s", epee::string_tools::pod_to_hex(get_block_hash(bei.bl)).c_str(), NULL); + for (const auto& notifier : m_block_notifiers) + { + std::size_t notify_height = split_height; + for (const auto& bei: alt_chain) + { + notifier(notify_height, {std::addressof(bei.bl), 1}); + ++notify_height; + } + } MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height()); return true; @@ -1296,7 +1301,7 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: size_t count = 0; size_t max_i = timestamps.size()-1; // get difficulties and timestamps from most recent blocks in alt chain - for (const auto bei: boost::adaptors::reverse(alt_chain)) + for (const auto &bei: boost::adaptors::reverse(alt_chain)) { timestamps[max_i - count] = bei.bl.timestamp; cumulative_difficulties[max_i - count] = bei.cumulative_difficulty; @@ -2153,7 +2158,7 @@ bool Blockchain::get_alternative_blocks(std::vector<block>& blocks) const CRITICAL_REGION_LOCAL(m_blockchain_lock); blocks.reserve(m_db->get_alt_block_count()); - m_db->for_all_alt_blocks([&blocks](const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata *blob) { + m_db->for_all_alt_blocks([&blocks](const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref *blob) { if (!blob) { MERROR("No blob, but blobs were requested"); @@ -4236,12 +4241,9 @@ leave: get_difficulty_for_next_block(); // just to cache it invalidate_block_template_cache(); - if (notify) - { - std::shared_ptr<tools::Notify> block_notify = m_block_notify; - if (block_notify) - block_notify->notify("%s", epee::string_tools::pod_to_hex(id).c_str(), NULL); - } + + for (const auto& notifier: m_block_notifiers) + notifier(new_height - 1, {std::addressof(bl), 1}); return true; } @@ -4369,6 +4371,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti //------------------------------------------------------------------ bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc) { + try + { + LOG_PRINT_L3("Blockchain::" << __func__); crypto::hash id = get_block_hash(bl); CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process @@ -4396,6 +4401,14 @@ bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc) rtxn_guard.stop(); return handle_block_to_main_chain(bl, id, bvc); + + } + catch (const std::exception &e) + { + LOG_ERROR("Exception at [add_new_block], what=" << e.what()); + bvc.m_verifivation_failed = true; + return false; + } } //------------------------------------------------------------------ //TODO: Refactor, consider returning a failure height and letting @@ -5108,7 +5121,7 @@ cryptonote::blobdata Blockchain::get_txpool_tx_blob(const crypto::hash& txid, re return m_db->get_txpool_tx_blob(txid, tx_category); } -bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category tx_category) const +bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob, relay_category tx_category) const { return m_db->for_all_txpool_txes(f, include_blob, tx_category); } @@ -5132,6 +5145,15 @@ void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint m_max_prepare_blocks_threads = maxthreads; } +void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)>&& notify) +{ + if (notify) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_block_notifiers.push_back(std::move(notify)); + } +} + void Blockchain::safesyncmode(const bool onoff) { /* all of this is no-op'd if the user set a specific @@ -5170,7 +5192,7 @@ std::vector<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>> blocks_ext_by_hash alt_blocks; alt_blocks.reserve(m_db->get_alt_block_count()); - m_db->for_all_alt_blocks([&alt_blocks](const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata *blob) { + m_db->for_all_alt_blocks([&alt_blocks](const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref *blob) { if (!blob) { MERROR("No blob, but blobs were requested"); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index fb7e5c4f8..6c777ec82 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -30,6 +30,7 @@ #pragma once #include <boost/asio/io_service.hpp> +#include <boost/function/function_fwd.hpp> #include <boost/serialization/serialization.hpp> #include <boost/serialization/version.hpp> #include <boost/serialization/list.hpp> @@ -764,7 +765,7 @@ namespace cryptonote * * @param notify the notify object to call at every new block */ - void set_block_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_block_notify = notify; } + void add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)> &¬ify); /** * @brief sets a reorg notify object to call for every reorg @@ -984,7 +985,7 @@ namespace cryptonote bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const; cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const; - bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category tx_category = relay_category::broadcasted) const; + bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category tx_category = relay_category::broadcasted) const; bool txpool_tx_matches_category(const crypto::hash& tx_hash, relay_category category); bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } @@ -1125,7 +1126,11 @@ namespace cryptonote bool m_batch_success; - std::shared_ptr<tools::Notify> m_block_notify; + /* `boost::function` is used because the implementation never allocates if + the callable object has a single `std::shared_ptr` or `std::weap_ptr` + internally. Whereas, the libstdc++ `std::function` will allocate. */ + + std::vector<boost::function<void(std::uint64_t, epee::span<const block>)>> m_block_notifiers; std::shared_ptr<tools::Notify> m_reorg_notify; // for prepare_handle_incoming_blocks diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 141e54459..9a1439c4a 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -41,6 +41,7 @@ using namespace epee; #include "common/download.h" #include "common/threadpool.h" #include "common/command_line.h" +#include "cryptonote_basic/events.h" #include "warnings.h" #include "crypto/crypto.h" #include "cryptonote_config.h" @@ -51,6 +52,7 @@ using namespace epee; #include "ringct/rctTypes.h" #include "blockchain_db/blockchain_db.h" #include "ringct/rctSigs.h" +#include "rpc/zmq_pub.h" #include "common/notify.h" #include "hardforks/hardforks.h" #include "version.h" @@ -262,6 +264,13 @@ namespace cryptonote { m_blockchain_storage.set_enforce_dns_checkpoints(enforce_dns); } + //----------------------------------------------------------------------------------- + void core::set_txpool_listener(boost::function<void(std::vector<txpool_event>)> zmq_pub) + { + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + m_zmq_pub = std::move(zmq_pub); + } + //----------------------------------------------------------------------------------------------- bool core::update_checkpoints(const bool skip_dns /* = false */) { @@ -614,7 +623,20 @@ namespace cryptonote try { if (!command_line::is_arg_defaulted(vm, arg_block_notify)) - m_blockchain_storage.set_block_notify(std::shared_ptr<tools::Notify>(new tools::Notify(command_line::get_arg(vm, arg_block_notify).c_str()))); + { + struct hash_notify + { + tools::Notify cmdline; + + void operator()(std::uint64_t, epee::span<const block> blocks) const + { + for (const block bl : blocks) + cmdline.notify("%s", epee::string_tools::pod_to_hex(get_block_hash(bl)).c_str(), NULL); + } + }; + + m_blockchain_storage.add_block_notify(hash_notify{{command_line::get_arg(vm, arg_block_notify).c_str()}}); + } } catch (const std::exception &e) { @@ -957,8 +979,7 @@ namespace cryptonote return false; } - struct result { bool res; cryptonote::transaction tx; crypto::hash hash; }; - std::vector<result> results(tx_blobs.size()); + std::vector<txpool_event> results(tx_blobs.size()); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); @@ -1023,6 +1044,7 @@ namespace cryptonote if (!tx_info.empty()) handle_incoming_tx_accumulated_batch(tx_info, tx_relay == relay_method::block); + bool valid_events = false; bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { @@ -1045,10 +1067,18 @@ namespace cryptonote {MERROR_VER("Transaction verification impossible: " << results[i].hash);} if(tvc[i].m_added_to_pool) + { MDEBUG("tx added: " << results[i].hash); + valid_events = true; + } + else + results[i].res = false; } - return ok; + if (valid_events && m_zmq_pub && matches_category(tx_relay, relay_category::legacy)) + m_zmq_pub(std::move(results)); + + return ok; CATCH_ENTRY_L0("core::handle_incoming_txs()", false); } //----------------------------------------------------------------------------------------------- @@ -1273,6 +1303,7 @@ namespace cryptonote { NOTIFY_NEW_TRANSACTIONS::request public_req{}; NOTIFY_NEW_TRANSACTIONS::request private_req{}; + NOTIFY_NEW_TRANSACTIONS::request stem_req{}; for (auto& tx : txs) { switch (std::get<2>(tx)) @@ -1283,6 +1314,9 @@ namespace cryptonote case relay_method::local: private_req.txs.push_back(std::move(std::get<1>(tx))); break; + case relay_method::forward: + stem_req.txs.push_back(std::move(std::get<1>(tx))); + break; case relay_method::block: case relay_method::fluff: case relay_method::stem: @@ -1300,6 +1334,8 @@ namespace cryptonote get_protocol()->relay_transactions(public_req, source, epee::net_utils::zone::public_, relay_method::fluff); if (!private_req.txs.empty()) get_protocol()->relay_transactions(private_req, source, epee::net_utils::zone::invalid, relay_method::local); + if (!stem_req.txs.empty()) + get_protocol()->relay_transactions(stem_req, source, epee::net_utils::zone::public_, relay_method::stem); } return true; } @@ -1649,9 +1685,8 @@ namespace cryptonote << "You can set the level of process detailization through \"set_log <level|categories>\" command," << ENDL << "where <level> is between 0 (no details) and 4 (very verbose), or custom category based levels (eg, *:WARNING)." << ENDL << ENDL - << "Use the \"help\" command to see a simplified list of available commands." << ENDL - << "Use the \"help_advanced\" command to see an advanced list of available commands." << ENDL - << "Use \"help_advanced <command>\" to see a command's documentation." << ENDL + << "Use the \"help\" command to see the list of available commands." << ENDL + << "Use \"help <command>\" to see a command's documentation." << ENDL << "**********************************************************************" << ENDL); m_starter_message_showed = true; } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 6a9ffda92..a53596c2c 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -32,9 +32,11 @@ #include <ctime> +#include <boost/function.hpp> #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> +#include "cryptonote_basic/fwd.h" #include "cryptonote_core/i_core_events.h" #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "cryptonote_protocol/enums.h" @@ -48,6 +50,7 @@ #include "warnings.h" #include "crypto/hash.h" #include "span.h" +#include "rpc/fwd.h" PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -446,6 +449,13 @@ namespace cryptonote void set_enforce_dns_checkpoints(bool enforce_dns); /** + * @brief set a listener for txes being added to the txpool + * + * @param callable to notify, or empty function to disable. + */ + void set_txpool_listener(boost::function<void(std::vector<txpool_event>)> zmq_pub); + + /** * @brief set whether or not to enable or disable DNS checkpoints * * @param disble whether to disable DNS checkpoints @@ -1098,7 +1108,12 @@ namespace cryptonote bool m_fluffy_blocks_enabled; bool m_offline; + /* `boost::function` is used because the implementation never allocates if + the callable object has a single `std::shared_ptr` or `std::weap_ptr` + internally. Whereas, the libstdc++ `std::function` will allocate. */ + std::shared_ptr<tools::Notify> m_block_rate_notify; + boost::function<void(std::vector<txpool_event>)> m_zmq_pub; }; } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 74aab88c4..85bcf2246 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -91,6 +91,8 @@ namespace cryptonote time_t const MAX_RELAY_TIME = (60 * 60 * 4); // at most that many seconds between resends float const ACCEPT_THRESHOLD = 1.0f; + constexpr const std::chrono::seconds forward_delay_average{CRYPTONOTE_FORWARD_DELAY_AVERAGE}; + // a kind of increasing backoff within min/max bounds uint64_t get_relay_delay(time_t now, time_t received) { @@ -309,8 +311,14 @@ namespace cryptonote if (meta.upgrade_relay_method(tx_relay) || !existing_tx) // synchronize with embargo timer or stem/fluff out-of-order messages { + using clock = std::chrono::system_clock; + auto last_relayed_time = std::numeric_limits<decltype(meta.last_relayed_time)>::max(); + if (tx_relay == relay_method::forward) + last_relayed_time = clock::to_time_t(clock::now() + crypto::random_poisson_seconds{forward_delay_average}()); + // else the `set_relayed` function will adjust the time accordingly later + //update transactions container - meta.last_relayed_time = std::numeric_limits<decltype(meta.last_relayed_time)>::max(); + meta.last_relayed_time = last_relayed_time; meta.receive_time = receive_time; meta.weight = tx_weight; meta.fee = fee; @@ -341,7 +349,7 @@ namespace cryptonote tvc.m_added_to_pool = true; static_assert(unsigned(relay_method::none) == 0, "expected relay_method::none value to be zero"); - if(meta.fee > 0) + if(meta.fee > 0 && tx_relay != relay_method::forward) tvc.m_relay = tx_relay; } @@ -614,7 +622,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_blockchain.for_all_txpool_txes([this, &hashes, &txes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_blockchain.for_all_txpool_txes([this, &hashes, &txes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*) { const auto tx_relay_method = meta.get_relay_method(); if (tx_relay_method != relay_method::block && tx_relay_method != relay_method::fluff) return true; @@ -662,7 +670,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); std::list<std::pair<crypto::hash, uint64_t>> remove; - m_blockchain.for_all_txpool_txes([this, &remove](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_blockchain.for_all_txpool_txes([this, &remove](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*) { uint64_t tx_age = time(nullptr) - meta.receive_time; if((tx_age > CRYPTONOTE_MEMPOOL_TX_LIVETIME && !meta.kept_by_block) || @@ -722,28 +730,46 @@ namespace cryptonote //TODO: investigate whether boolean return is appropriate bool tx_memory_pool::get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>> &txs) const { + std::vector<std::pair<crypto::hash, txpool_tx_meta_t>> change_timestamps; + const uint64_t now = time(NULL); + CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - const uint64_t now = time(NULL); + LockedTXN lock(m_blockchain.get_db()); txs.reserve(m_blockchain.get_txpool_tx_count()); - m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ + m_blockchain.for_all_txpool_txes([this, now, &txs, &change_timestamps](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *){ // 0 fee transactions are never relayed if(!meta.pruned && meta.fee > 0 && !meta.do_not_relay) { - if (!meta.dandelionpp_stem && now - meta.last_relayed_time <= get_relay_delay(now, meta.receive_time)) - return true; - if (meta.dandelionpp_stem && meta.last_relayed_time < now) // for dandelion++ stem, this value is the embargo timeout - return true; + const relay_method tx_relay = meta.get_relay_method(); + switch (tx_relay) + { + case relay_method::stem: + case relay_method::forward: + if (meta.last_relayed_time > now) + return true; // continue to next tx + change_timestamps.emplace_back(txid, meta); + break; + default: + case relay_method::none: + return true; + case relay_method::local: + case relay_method::fluff: + case relay_method::block: + if (now - meta.last_relayed_time <= get_relay_delay(now, meta.receive_time)) + return true; // continue to next tx + break; + } // if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem // mentioned by smooth where nodes would flush txes at slightly different times, causing // flushed txes to be re-added when received from a node which was just about to flush it - uint64_t max_age = meta.kept_by_block ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME; + uint64_t max_age = (tx_relay == relay_method::block) ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME; if (now - meta.receive_time <= max_age / 2) { try { - txs.emplace_back(txid, m_blockchain.get_txpool_tx_blob(txid, relay_category::all), meta.get_relay_method()); + txs.emplace_back(txid, m_blockchain.get_txpool_tx_blob(txid, relay_category::all), tx_relay); } catch (const std::exception &e) { @@ -754,6 +780,18 @@ namespace cryptonote } return true; }, false, relay_category::relayable); + + for (auto& elem : change_timestamps) + { + /* These transactions are still in forward or stem state, so the field + represents the next time a relay should be attempted. Will be + overwritten when the state is upgraded to stem, fluff or block. This + function is only called every ~2 minutes, so this resetting should be + unnecessary, but is primarily a precaution against potential changes + to the callback routines. */ + elem.second.last_relayed_time = now + get_relay_delay(now, elem.second.receive_time); + m_blockchain.update_txpool_tx(elem.first, elem.second); + } return true; } //--------------------------------------------------------------------------------- @@ -806,7 +844,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; txs.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); - m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ transaction tx; if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) { @@ -826,7 +864,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; txs.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); - m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ txs.push_back(txid); return true; }, false, category); @@ -839,7 +877,7 @@ namespace cryptonote const uint64_t now = time(NULL); const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); - m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; }, false, category); @@ -855,7 +893,7 @@ namespace cryptonote stats.txs_total = m_blockchain.get_txpool_tx_count(include_sensitive); std::vector<uint32_t> weights; weights.reserve(stats.txs_total); - m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ weights.push_back(meta.weight); stats.bytes_total += meta.weight; if (!stats.bytes_min || meta.weight < stats.bytes_min) @@ -940,10 +978,10 @@ namespace cryptonote const size_t count = m_blockchain.get_txpool_tx_count(include_sensitive_data); tx_infos.reserve(count); key_image_infos.reserve(count); - m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); - txi.tx_blob = *bd; + txi.tx_blob = blobdata(bd->data(), bd->size()); transaction tx; if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) { @@ -997,7 +1035,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); tx_infos.reserve(m_blockchain.get_txpool_tx_count()); key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); - m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, txi.tx) : parse_and_validate_tx_from_blob(*bd, txi.tx))) @@ -1293,7 +1331,7 @@ namespace cryptonote std::stringstream ss; CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_blockchain.for_all_txpool_txes([&ss, short_format](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *txblob) { + m_blockchain.for_all_txpool_txes([&ss, short_format](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *txblob) { ss << "id: " << txid << std::endl; if (!short_format) { cryptonote::transaction tx; @@ -1471,7 +1509,7 @@ namespace cryptonote std::unordered_set<crypto::hash> remove; m_txpool_weight = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*) { m_txpool_weight += meta.weight; if (meta.weight > tx_weight_limit) { LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); @@ -1543,7 +1581,7 @@ namespace cryptonote for (int pass = 0; pass < 2; ++pass) { const bool kept = pass == 1; - bool r = m_blockchain.for_all_txpool_txes([this, &remove, kept](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd) { + bool r = m_blockchain.for_all_txpool_txes([this, &remove, kept](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd) { if (!!kept != !!meta.kept_by_block) return true; cryptonote::transaction_prefix tx; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 02c416af5..337885509 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -935,7 +935,19 @@ namespace cryptonote return 1; } - relay_method tx_relay; + /* If the txes were received over i2p/tor, the default is to "forward" + with a randomized delay to further enhance the "white noise" behavior, + potentially making it harder for ISP-level spies to determine which + inbound link sent the tx. If the sender disabled "white noise" over + i2p/tor, then the sender is "fluffing" (to only outbound) i2p/tor + connections with the `dandelionpp_fluff` flag set. The receiver (hidden + service) will immediately fluff in that scenario (i.e. this assumes that a + sybil spy will be unable to link an IP to an i2p/tor connection). */ + + const epee::net_utils::zone zone = context.m_remote_address.get_zone(); + relay_method tx_relay = zone == epee::net_utils::zone::public_ ? + relay_method::stem : relay_method::forward; + std::vector<blobdata> stem_txs{}; std::vector<blobdata> fluff_txs{}; if (arg.dandelionpp_fluff) @@ -944,10 +956,7 @@ namespace cryptonote fluff_txs.reserve(arg.txs.size()); } else - { - tx_relay = relay_method::stem; stem_txs.reserve(arg.txs.size()); - } for (auto& tx : arg.txs) { @@ -970,6 +979,7 @@ namespace cryptonote fluff_txs.push_back(std::move(tx)); break; default: + case relay_method::forward: // not supposed to happen here case relay_method::none: break; } @@ -1741,7 +1751,6 @@ skip: if(!m_core.find_blockchain_supplement(arg.block_ids, !arg.prune, r)) { LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN."); - drop_connection(context, false, false); return 1; } MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); @@ -1928,8 +1937,8 @@ skip: if (local_stripe == 0) return false; // don't request pre-bulletprooof pruned blocks, we can't reconstruct their weight (yet) - static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(8); - if (first_block_height + nblocks - 1 < bp_fork_height) + static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(HF_VERSION_SMALLER_BP); + if (first_block_height < bp_fork_height) return false; // assumes the span size is less or equal to the stripe size bool full_data_needed = tools::get_pruning_stripe(first_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe @@ -2338,8 +2347,7 @@ skip: MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL << "You are now synchronized with the network. You may now start monero-wallet-cli." << ENDL << ENDL - << "Use the \"help\" command to see a simplified list of available commands." << ENDL - << "Use the \"help_advanced\" command to see an advanced list of available commands." << ENDL + << "Use the \"help\" command to see the list of available commands." << ENDL << "**********************************************************************"); m_sync_timer.pause(); if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) diff --git a/src/cryptonote_protocol/enums.h b/src/cryptonote_protocol/enums.h index fabb82c61..c0c495837 100644 --- a/src/cryptonote_protocol/enums.h +++ b/src/cryptonote_protocol/enums.h @@ -37,6 +37,7 @@ namespace cryptonote { none = 0, //!< Received via RPC with `do_not_relay` set local, //!< Received via RPC; trying to send over i2p/tor, etc. + forward, //!< Received over i2p/tor; timer delayed before ipv4/6 public broadcast stem, //!< Received/send over network using Dandelion++ stem fluff, //!< Received/sent over network using Dandelion++ fluff block //!< Received in block, takes precedence over others diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 56181a59b..7c482156f 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -357,11 +357,15 @@ namespace levin return true; }); - // Always send txs in stem mode over i2p/tor, see comments in `send_txs` below. + /* Always send with `fluff` flag, even over i2p/tor. The hidden service + will disable the forwarding delay and immediately fluff. The i2p/tor + network is therefore replacing the sybil protection of Dandelion++. + Dandelion++ stem phase over i2p/tor is also worth investigating + (with/without "noise"?). */ for (auto& connection : connections) { std::sort(connection.first.begin(), connection.first.end()); // don't leak receive order - make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs, zone_->is_public); + make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs, true); } if (next_flush != std::chrono::steady_clock::time_point::max()) @@ -811,12 +815,11 @@ namespace levin case relay_method::block: return false; case relay_method::stem: - tx_relay = relay_method::fluff; // don't set stempool embargo when skipping to fluff - /* fallthrough */ + case relay_method::forward: case relay_method::local: if (zone_->is_public) { - // this will change a local tx to stem or fluff ... + // this will change a local/forward tx to stem or fluff ... zone_->strand.dispatch( dandelionpp_notify{zone_, std::addressof(core), std::move(txs), source} ); @@ -824,6 +827,11 @@ namespace levin } /* fallthrough */ case relay_method::fluff: + /* If sending stem/forward/local txes over non public networks, + continue to claim that relay mode even though it used the "fluff" + routine. A "fluff" over i2p/tor is not the same as a "fluff" over + ipv4/6. Marking it as "fluff" here will make the tx immediately + visible externally from this node, which is not desired. */ core.on_transactions_relayed(epee::to_span(txs), tx_relay); zone_->strand.dispatch(fluff_notify{zone_, std::move(txs), source}); break; diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 0ce987bcc..6c3e163e6 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -121,6 +121,10 @@ namespace daemon_args return val; } }; + const command_line::arg_descriptor<std::vector<std::string>> arg_zmq_pub = { + "zmq-pub" + , "Address for ZMQ pub - tcp://ip:port or ipc://path" + }; const command_line::arg_descriptor<bool> arg_zmq_rpc_disabled = { "no-zmq" diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index b335116de..ac4c30726 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -58,6 +58,12 @@ t_command_server::t_command_server( , "Show the help section or the documentation about a <command>." ); m_command_lookup.set_handler( + "apropos" + , std::bind(&t_command_server::apropos, this, p::_1) + , "apropos <keyword> [<keyword> ...]" + , "Search all command descriptions for keyword(s)." + ); + m_command_lookup.set_handler( "print_height" , std::bind(&t_command_parser_executor::print_height, &m_parser, p::_1) , "Print the local blockchain height." @@ -349,7 +355,7 @@ bool t_command_server::start_handling(std::function<void(void)> exit_handler) { if (m_is_rpc) return false; - m_command_lookup.start_handling("", get_commands_str(), exit_handler); + m_command_lookup.start_handling("", "Use \"help\" to list all commands and their usage\n", exit_handler); return true; } @@ -374,6 +380,33 @@ bool t_command_server::help(const std::vector<std::string>& args) return true; } +bool t_command_server::apropos(const std::vector<std::string>& args) +{ + if (args.empty()) + { + std::cout << "Missing keyword" << std::endl; + return true; + } + const std::vector<std::string>& command_list = m_command_lookup.get_command_list(args); + if (command_list.empty()) + { + std::cout << "Nothing found" << std::endl; + return true; + } + + std::cout << std::endl; + for(auto const& command:command_list) + { + std::vector<std::string> cmd; + cmd.push_back(command); + std::pair<std::string, std::string> documentation = m_command_lookup.get_documentation(cmd); + std::cout << " " << documentation.first << std::endl; + } + std::cout << std::endl; + + return true; +} + std::string t_command_server::get_commands_str() { std::stringstream ss; @@ -382,7 +415,7 @@ std::string t_command_server::get_commands_str() std::string usage = m_command_lookup.get_usage(); boost::replace_all(usage, "\n", "\n "); usage.insert(0, " "); - ss << usage << std::endl; + ss << usage; return ss.str(); } diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h index 946d55b8c..df7198d04 100644 --- a/src/daemon/command_server.h +++ b/src/daemon/command_server.h @@ -73,6 +73,7 @@ public: private: bool help(const std::vector<std::string>& args); + bool apropos(const std::vector<std::string>& args); std::string get_commands_str(); std::string get_command_usage(const std::vector<std::string> &args); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 96db8712b..99430b2b0 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -34,10 +34,12 @@ #include "misc_log_ex.h" #include "daemon/daemon.h" #include "rpc/daemon_handler.h" +#include "rpc/zmq_pub.h" #include "rpc/zmq_server.h" #include "common/password.h" #include "common/util.h" +#include "cryptonote_basic/events.h" #include "daemon/core.h" #include "daemon/p2p.h" #include "daemon/protocol.h" @@ -56,6 +58,17 @@ using namespace epee; namespace daemonize { +struct zmq_internals +{ + explicit zmq_internals(t_core& core, t_p2p& p2p) + : rpc_handler{core.get(), p2p.get()} + , server{rpc_handler} + {} + + cryptonote::rpc::DaemonHandler rpc_handler; + cryptonote::rpc::ZmqServer server; +}; + struct t_internals { private: t_protocol protocol; @@ -63,6 +76,7 @@ public: t_core core; t_p2p p2p; std::vector<std::unique_ptr<t_rpc>> rpcs; + std::unique_ptr<zmq_internals> zmq; t_internals( boost::program_options::variables_map const & vm @@ -70,6 +84,7 @@ public: : core{vm} , protocol{vm, core, command_line::get_arg(vm, cryptonote::arg_offline)} , p2p{vm, protocol} + , zmq{nullptr} { // Handle circular dependencies protocol.set_p2p_endpoint(p2p.get()); @@ -86,6 +101,28 @@ public: auto restricted_rpc_port = command_line::get_arg(vm, restricted_rpc_port_arg); rpcs.emplace_back(new t_rpc{vm, core, p2p, true, restricted_rpc_port, "restricted", true}); } + + if (!command_line::get_arg(vm, daemon_args::arg_zmq_rpc_disabled)) + { + zmq.reset(new zmq_internals{core, p2p}); + + const std::string zmq_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port); + const std::string zmq_address = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_ip); + + if (!zmq->server.init_rpc(zmq_address, zmq_port)) + throw std::runtime_error{"Failed to add TCP socket(" + zmq_address + ":" + zmq_port + ") to ZMQ RPC Server"}; + + std::shared_ptr<cryptonote::listener::zmq_pub> shared; + const std::vector<std::string> zmq_pub = command_line::get_arg(vm, daemon_args::arg_zmq_pub); + if (!zmq_pub.empty() && !(shared = zmq->server.init_pub(epee::to_span(zmq_pub)))) + throw std::runtime_error{"Failed to initialize zmq_pub"}; + + if (shared) + { + core.get().get_blockchain_storage().add_block_notify(cryptonote::listener::zmq_pub::chain_main{shared}); + core.get().set_txpool_listener(cryptonote::listener::zmq_pub::txpool_add{shared}); + } + } } }; @@ -103,9 +140,6 @@ t_daemon::t_daemon( : mp_internals{new t_internals{vm}}, public_rpc_port(public_rpc_port) { - zmq_rpc_bind_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port); - zmq_rpc_bind_address = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_ip); - zmq_rpc_disabled = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_disabled); } t_daemon::~t_daemon() = default; @@ -169,31 +203,8 @@ bool t_daemon::run(bool interactive) rpc_commands->start_handling(std::bind(&daemonize::t_daemon::stop_p2p, this)); } - cryptonote::rpc::DaemonHandler rpc_daemon_handler(mp_internals->core.get(), mp_internals->p2p.get()); - cryptonote::rpc::ZmqServer zmq_server(rpc_daemon_handler); - - if (!zmq_rpc_disabled) - { - if (!zmq_server.addTCPSocket(zmq_rpc_bind_address, zmq_rpc_bind_port)) - { - LOG_ERROR(std::string("Failed to add TCP Socket (") + zmq_rpc_bind_address - + ":" + zmq_rpc_bind_port + ") to ZMQ RPC Server"); - - if (rpc_commands) - rpc_commands->stop_handling(); - - for(auto& rpc : mp_internals->rpcs) - rpc->stop(); - - return false; - } - - MINFO("Starting ZMQ server..."); - zmq_server.run(); - - MINFO(std::string("ZMQ server started at ") + zmq_rpc_bind_address - + ":" + zmq_rpc_bind_port + "."); - } + if (mp_internals->zmq) + mp_internals->zmq->server.run(); else MINFO("ZMQ server disabled"); @@ -208,8 +219,8 @@ bool t_daemon::run(bool interactive) if (rpc_commands) rpc_commands->stop_handling(); - if (!zmq_rpc_disabled) - zmq_server.stop(); + if (mp_internals->zmq) + mp_internals->zmq->server.stop(); for(auto& rpc : mp_internals->rpcs) rpc->stop(); diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h index bb7fdfebd..2eb2019ce 100644 --- a/src/daemon/daemon.h +++ b/src/daemon/daemon.h @@ -44,9 +44,6 @@ private: private: std::unique_ptr<t_internals> mp_internals; uint16_t public_rpc_port; - std::string zmq_rpc_bind_address; - std::string zmq_rpc_bind_port; - bool zmq_rpc_disabled; public: t_daemon( boost::program_options::variables_map const & vm, diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index dfc35470e..f2ae6dcc3 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -154,6 +154,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_public_node); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); + command_line::add_arg(core_settings, daemon_args::arg_zmq_pub); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_disabled); daemonizer::init_options(hidden_options, visible_options); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 77dfe332f..adaa9bc0e 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -721,10 +721,11 @@ bool t_rpc_command_executor::print_net_stats() uint64_t average = seconds > 0 ? net_stats_res.total_bytes_in / seconds : 0; uint64_t limit = limit_res.limit_down * 1024; // convert to bytes, as limits are always kB/s double percent = (double)average / (double)limit * 100.0; - tools::success_msg_writer() << boost::format("Received %u bytes (%s) in %u packets, average %s/s = %.2f%% of the limit of %s/s") + tools::success_msg_writer() << boost::format("Received %u bytes (%s) in %u packets in %s, average %s/s = %.2f%% of the limit of %s/s") % net_stats_res.total_bytes_in % tools::get_human_readable_bytes(net_stats_res.total_bytes_in) % net_stats_res.total_packets_in + % tools::get_human_readable_timespan(seconds) % tools::get_human_readable_bytes(average) % percent % tools::get_human_readable_bytes(limit); @@ -732,10 +733,11 @@ bool t_rpc_command_executor::print_net_stats() average = seconds > 0 ? net_stats_res.total_bytes_out / seconds : 0; limit = limit_res.limit_up * 1024; percent = (double)average / (double)limit * 100.0; - tools::success_msg_writer() << boost::format("Sent %u bytes (%s) in %u packets, average %s/s = %.2f%% of the limit of %s/s") + tools::success_msg_writer() << boost::format("Sent %u bytes (%s) in %u packets in %s, average %s/s = %.2f%% of the limit of %s/s") % net_stats_res.total_bytes_out % tools::get_human_readable_bytes(net_stats_res.total_bytes_out) % net_stats_res.total_packets_out + % tools::get_human_readable_timespan(seconds) % tools::get_human_readable_bytes(average) % percent % tools::get_human_readable_bytes(limit); @@ -1001,7 +1003,9 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, if (1 == res.txs.size()) { // only available for new style answers - bool pruned = res.txs.front().prunable_as_hex.empty() && res.txs.front().prunable_hash != epee::string_tools::pod_to_hex(crypto::null_hash); + static const std::string empty_hash = epee::string_tools::pod_to_hex(crypto::cn_fast_hash("", 0)); + // prunable_hash will equal empty_hash when nothing is prunable (mostly when the transaction is coinbase) + bool pruned = res.txs.front().prunable_as_hex.empty() && res.txs.front().prunable_hash != epee::string_tools::pod_to_hex(crypto::null_hash) && res.txs.front().prunable_hash != empty_hash; if (res.txs.front().in_pool) tools::success_msg_writer() << "Found in pool"; else diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index 42dba2ebb..ff2afba4b 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -72,6 +72,7 @@ target_link_libraries(device ${HIDAPI_LIBRARIES} cncrypto ringct_basic + wallet-crypto ${OPENSSL_CRYPTO_LIBRARIES} ${Boost_SERIALIZATION_LIBRARY} PRIVATE diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 7e054af35..096cb35ba 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -32,6 +32,7 @@ #include "device_default.hpp" #include "int-util.h" +#include "crypto/wallet/crypto.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" #include "cryptonote_core/cryptonote_tx_utils.h" @@ -120,7 +121,7 @@ namespace hw { /* ======================================================================= */ bool device_default::derive_subaddress_public_key(const crypto::public_key &out_key, const crypto::key_derivation &derivation, const std::size_t output_index, crypto::public_key &derived_key) { - return crypto::derive_subaddress_public_key(out_key, derivation, output_index,derived_key); + return crypto::wallet::derive_subaddress_public_key(out_key, derivation, output_index,derived_key); } crypto::public_key device_default::get_subaddress_spend_public_key(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { @@ -236,7 +237,7 @@ namespace hw { } bool device_default::generate_key_derivation(const crypto::public_key &key1, const crypto::secret_key &key2, crypto::key_derivation &derivation) { - return crypto::generate_key_derivation(key1, key2, derivation); + return crypto::wallet::generate_key_derivation(key1, key2, derivation); } bool device_default::derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res){ diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 4bd3d75b1..30964848d 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1548,7 +1548,7 @@ namespace hw { const bool need_additional_txkeys_x = need_additional_txkeys; std::vector<crypto::secret_key> additional_tx_keys_x; - for (const auto k: additional_tx_keys) { + for (const auto &k: additional_tx_keys) { additional_tx_keys_x.push_back(hw::ledger::decrypt(k)); } diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 1ce7118ad..afcd42ef7 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,9 +26,9 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources dandelionpp.cpp error.cpp i2p_address.cpp parse.cpp socks.cpp +set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp zmq.cpp) -set(net_headers dandelionpp.h error.h i2p_address.h parse.h socks.h socks_connect.h +set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h socks_connect.h tor_address.h zmq.h) monero_add_library(net ${net_sources} ${net_headers}) diff --git a/src/serialization/set.h b/src/net/http.cpp index 157c002f7..f54b65c29 100644 --- a/src/serialization/set.h +++ b/src/net/http.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2020, The Monero Project -// +// Copyright (c) 2020, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL @@ -25,34 +25,46 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#pragma once +#include "http.h" -#include <set> +#include "parse.h" +#include "socks_connect.h" -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::set<T> &v); -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::set<T> &v); +namespace net +{ +namespace http +{ -namespace serialization +bool client::set_proxy(const std::string &address) { - namespace detail + if (address.empty()) + { + set_connector(epee::net_utils::direct_connect{}); + } + else { - template <typename T> - void do_add(std::set<T> &c, T &&e) + const auto endpoint = get_tcp_endpoint(address); + if (!endpoint) + { + auto always_fail = net::socks::connector{boost::asio::ip::tcp::endpoint()}; + set_connector(always_fail); + } + else { - c.insert(std::move(e)); + set_connector(net::socks::connector{*endpoint}); } } -} -#include "serialization.h" + disconnect(); + + return true; +} -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::set<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::set<T> &v) { return do_serialize_container(ar, v); } +std::unique_ptr<epee::net_utils::http::abstract_http_client> client_factory::create() +{ + return std::unique_ptr<epee::net_utils::http::abstract_http_client>(new client()); +} +} // namespace http +} // namespace net diff --git a/src/net/http.h b/src/net/http.h new file mode 100644 index 000000000..91a345851 --- /dev/null +++ b/src/net/http.h @@ -0,0 +1,51 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "net/http_client.h" + +namespace net +{ +namespace http +{ + +class client : public epee::net_utils::http::http_simple_client +{ +public: + bool set_proxy(const std::string &address) override; +}; + +class client_factory : public epee::net_utils::http::http_client_factory +{ +public: + std::unique_ptr<epee::net_utils::http::abstract_http_client> create() override; +}; + +} // namespace http +} // namespace net diff --git a/src/net/parse.cpp b/src/net/parse.cpp index ce580afe7..95b22cd44 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -122,4 +122,39 @@ namespace net return {epee::net_utils::ipv4_network_subnet{ip, (uint8_t)mask}}; } + + expect<boost::asio::ip::tcp::endpoint> get_tcp_endpoint(const boost::string_ref address) + { + uint16_t port = 0; + expect<epee::net_utils::network_address> parsed = get_network_address(address, port); + if (!parsed) + { + return parsed.error(); + } + + boost::asio::ip::tcp::endpoint result; + switch (parsed->get_type_id()) + { + case epee::net_utils::ipv4_network_address::get_type_id(): + { + const auto &ipv4 = parsed->as<epee::net_utils::ipv4_network_address>(); + result = boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4(ipv4.ip()), ipv4.port()); + break; + } + case epee::net_utils::ipv6_network_address::get_type_id(): + { + const auto &ipv6 = parsed->as<epee::net_utils::ipv6_network_address>(); + result = boost::asio::ip::tcp::endpoint(ipv6.ip(), ipv6.port()); + break; + } + default: + return make_error_code(net::error::unsupported_address); + } + if (result.port() == 0) + { + return make_error_code(net::error::invalid_port); + } + + return result; + } } diff --git a/src/net/parse.h b/src/net/parse.h index 0d8fda711..4d5efe161 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -28,6 +28,7 @@ #pragma once +#include <boost/asio/ip/tcp.hpp> #include <boost/utility/string_ref.hpp> #include <cstdint> @@ -65,5 +66,7 @@ namespace net */ expect<epee::net_utils::ipv4_network_subnet> get_ipv4_subnet_address(boost::string_ref address, bool allow_implicit_32 = false); + + expect<boost::asio::ip::tcp::endpoint> get_tcp_endpoint(const boost::string_ref address); } diff --git a/src/net/zmq.cpp b/src/net/zmq.cpp index 1a0edb4b9..15560ca7e 100644 --- a/src/net/zmq.cpp +++ b/src/net/zmq.cpp @@ -158,20 +158,6 @@ namespace zmq return unsigned(max_out) < added ? max_out : int(added); } }; - - template<typename F, typename... T> - expect<void> retry_op(F op, T&&... args) noexcept(noexcept(op(args...))) - { - for (;;) - { - if (0 <= op(args...)) - return success(); - - const int error = zmq_errno(); - if (error != EINTR) - return make_error_code(error); - } - } } // anonymous expect<std::string> receive(void* const socket, const int flags) diff --git a/src/net/zmq.h b/src/net/zmq.h index 8c587ed7c..fa4ef2fc9 100644 --- a/src/net/zmq.h +++ b/src/net/zmq.h @@ -26,6 +26,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + #include <memory> #include <string> #include <system_error> @@ -105,6 +107,26 @@ namespace zmq //! Unique ZMQ socket handle, calls `zmq_close` on destruction. using socket = std::unique_ptr<void, close>; + /*! Retry a ZMQ function on `EINTR` errors. `F` must return an int with + values less than 0 on error. + + \param op The ZMQ function to execute + retry + \param args Forwarded to `op`. Must be resuable in case of retry. + \return All errors except for `EINTR`. */ + template<typename F, typename... T> + expect<void> retry_op(F op, T&&... args) noexcept(noexcept(op(args...))) + { + for (;;) + { + if (0 <= op(args...)) + return success(); + + const int error = zmq_errno(); + if (error != EINTR) + return make_error_code(error); + } + } + /*! Read all parts of the next message on `socket`. Blocks until the entire next message (all parts) are read, or until `zmq_term` is called on the `zmq_context` associated with `socket`. If the context is terminated, diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index fb3a38b07..175741146 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -604,16 +604,12 @@ namespace nodetool if (nettype == cryptonote::TESTNET) { full_addrs.insert("212.83.175.67:28080"); - full_addrs.insert("5.9.100.248:28080"); - full_addrs.insert("163.172.182.165:28080"); - full_addrs.insert("195.154.123.123:28080"); full_addrs.insert("212.83.172.165:28080"); full_addrs.insert("192.110.160.146:28080"); } else if (nettype == cryptonote::STAGENET) { full_addrs.insert("162.210.173.150:38080"); - full_addrs.insert("162.210.173.151:38080"); full_addrs.insert("192.110.160.146:38080"); } else if (nettype == cryptonote::FAKECHAIN) @@ -621,13 +617,7 @@ namespace nodetool } else { - full_addrs.insert("107.152.130.98:18080"); full_addrs.insert("212.83.175.67:18080"); - full_addrs.insert("5.9.100.248:18080"); - full_addrs.insert("163.172.182.165:18080"); - full_addrs.insert("161.67.132.39:18080"); - full_addrs.insert("198.74.231.92:18080"); - full_addrs.insert("195.154.123.123:18080"); full_addrs.insert("212.83.172.165:18080"); full_addrs.insert("192.110.160.146:18080"); full_addrs.insert("88.198.163.90:18080"); diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index bbc165cfa..b439dc47e 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -39,6 +39,7 @@ #include "misc_language.h" #include "string_tools.h" #include "time_helper.h" +#include "serialization/serialization.h" #include "cryptonote_config.h" namespace nodetool @@ -84,6 +85,15 @@ namespace nodetool KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) KV_SERIALIZE_OPT(rpc_credits_per_hash, (uint32_t)0) END_KV_SERIALIZE_MAP() + + BEGIN_SERIALIZE() + FIELD(adr) + FIELD(id) + VARINT_FIELD(last_seen) + VARINT_FIELD(pruning_seed) + VARINT_FIELD(rpc_port) + VARINT_FIELD(rpc_credits_per_hash) + END_SERIALIZE() }; typedef peerlist_entry_base<epee::net_utils::network_address> peerlist_entry; @@ -99,6 +109,12 @@ namespace nodetool KV_SERIALIZE(id) KV_SERIALIZE(first_seen) END_KV_SERIALIZE_MAP() + + BEGIN_SERIALIZE() + FIELD(adr) + FIELD(id) + VARINT_FIELD(first_seen) + END_SERIALIZE() }; typedef anchor_peerlist_entry_base<epee::net_utils::network_address> anchor_peerlist_entry; @@ -114,6 +130,12 @@ namespace nodetool KV_SERIALIZE(id) KV_SERIALIZE(is_income) END_KV_SERIALIZE_MAP() + + BEGIN_SERIALIZE() + FIELD(adr) + FIELD(id) + FIELD(is_income) + END_SERIALIZE() }; typedef connection_entry_base<epee::net_utils::network_address> connection_entry; diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 9b7f26a02..ce11981ad 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -49,7 +49,7 @@ extern "C" { #include "hex.h" #include "span.h" #include "memwipe.h" -#include "serialization/vector.h" +#include "serialization/containers.h" #include "serialization/debug_archive.h" #include "serialization/binary_archive.h" #include "serialization/json_archive.h" @@ -239,6 +239,12 @@ namespace rct { struct RCTConfig { RangeProofType range_proof_type; int bp_version; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + VARINT_FIELD(range_proof_type) + VARINT_FIELD(bp_version) + END_SERIALIZE() }; struct rctSigBase { uint8_t type; @@ -317,6 +323,16 @@ namespace rct { ar.end_array(); return ar.stream().good(); } + + BEGIN_SERIALIZE_OBJECT() + FIELD(type) + FIELD(message) + FIELD(mixRing) + FIELD(pseudoOuts) + FIELD(ecdhInfo) + FIELD(outPk) + VARINT_FIELD(txnFee) + END_SERIALIZE() }; struct rctSigPrunable { std::vector<rangeSig> rangeSigs; @@ -436,6 +452,12 @@ namespace rct { return ar.stream().good(); } + BEGIN_SERIALIZE_OBJECT() + FIELD(rangeSigs) + FIELD(bulletproofs) + FIELD(MGs) + FIELD(pseudoOuts) + END_SERIALIZE() }; struct rctSig: public rctSigBase { rctSigPrunable p; @@ -449,6 +471,11 @@ namespace rct { { return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } + + BEGIN_SERIALIZE_OBJECT() + FIELDS((rctSigBase&)*this) + FIELD(p) + END_SERIALIZE() }; //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 35195bd98..19298c969 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -45,8 +45,11 @@ set(daemon_messages_sources message.cpp daemon_messages.cpp) +set(rpc_pub_sources zmq_pub.cpp) + set(daemon_rpc_server_sources daemon_handler.cpp + zmq_pub.cpp zmq_server.cpp) @@ -59,8 +62,9 @@ set(rpc_headers rpc_version_str.h rpc_handler.h) -set(daemon_rpc_server_headers) +set(rpc_pub_headers zmq_pub.h) +set(daemon_rpc_server_headers) set(rpc_daemon_private_headers bootstrap_daemon.h @@ -83,6 +87,8 @@ set(daemon_rpc_server_private_headers monero_private_headers(rpc ${rpc_private_headers}) +set(rpc_pub_private_headers) + monero_private_headers(daemon_rpc_server ${daemon_rpc_server_private_headers}) @@ -97,6 +103,11 @@ monero_add_library(rpc ${rpc_headers} ${rpc_private_headers}) +monero_add_library(rpc_pub + ${rpc_pub_sources} + ${rpc_pub_headers} + ${rpc_pub_private_headers}) + monero_add_library(daemon_messages ${daemon_messages_sources} ${daemon_messages_headers} @@ -131,6 +142,14 @@ target_link_libraries(rpc PRIVATE ${EXTRA_LIBRARIES}) +target_link_libraries(rpc_pub + PUBLIC + epee + net + cryptonote_basic + serialization + ${Boost_THREAD_LIBRARY}) + target_link_libraries(daemon_messages LINK_PRIVATE cryptonote_core @@ -142,6 +161,7 @@ target_link_libraries(daemon_messages target_link_libraries(daemon_rpc_server LINK_PRIVATE rpc + rpc_pub cryptonote_core cryptonote_protocol version diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index bc561abc4..62822cfa3 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -771,7 +771,8 @@ namespace cryptonote outkey.mask = epee::string_tools::pod_to_hex(i.mask); outkey.unlocked = i.unlocked; outkey.height = i.height; - outkey.txid = epee::string_tools::pod_to_hex(i.txid); + if (req.get_txid) + outkey.txid = epee::string_tools::pod_to_hex(i.txid); } res.status = CORE_RPC_STATUS_OK; @@ -2453,14 +2454,13 @@ namespace cryptonote { for (const auto &str: req.txids) { - cryptonote::blobdata txid_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(str, txid_data)) + crypto::hash txid; + if(!epee::string_tools::hex_to_pod(str, txid)) { failed = true; } else { - crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); txids.push_back(txid); } } @@ -2805,15 +2805,14 @@ namespace cryptonote res.status = ""; for (const auto &str: req.txids) { - cryptonote::blobdata txid_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(str, txid_data)) + crypto::hash txid; + if(!epee::string_tools::hex_to_pod(str, txid)) { if (!res.status.empty()) res.status += ", "; res.status += std::string("invalid transaction id: ") + str; failed = true; continue; } - crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data()); cryptonote::blobdata txblob; if (m_core.get_pool_transaction(txid, txblob, relay_category::legacy)) diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index ab28319cb..0a26a4d5d 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -182,6 +182,7 @@ namespace rpc for (const auto& blob : it->second) { bwt.transactions.emplace_back(); + bwt.transactions.back().pruned = req.prune; if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back())) { res.blocks.clear(); @@ -905,13 +906,13 @@ namespace rpc return true; } - epee::byte_slice DaemonHandler::handle(const std::string& request) + epee::byte_slice DaemonHandler::handle(std::string&& request) { MDEBUG("Handling RPC request: " << request); try { - FullMessage req_full(request, true); + FullMessage req_full(std::move(request), true); const std::string request_type = req_full.getRequestType(); const auto matched_handler = std::lower_bound(std::begin(handlers), std::end(handlers), request_type); diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index aa3470c25..31c4f3ec4 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -133,7 +133,7 @@ class DaemonHandler : public RpcHandler void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); - epee::byte_slice handle(const std::string& request) override final; + epee::byte_slice handle(std::string&& request) override final; private: diff --git a/src/rpc/daemon_rpc_version.h b/src/rpc/daemon_rpc_version.h index d178a82bc..2955d5449 100644 --- a/src/rpc/daemon_rpc_version.h +++ b/src/rpc/daemon_rpc_version.h @@ -35,7 +35,7 @@ namespace rpc { static const uint32_t DAEMON_RPC_VERSION_ZMQ_MINOR = 0; -static const uint32_t DAEMON_RPC_VERSION_ZMQ_MAJOR = 1; +static const uint32_t DAEMON_RPC_VERSION_ZMQ_MAJOR = 2; static const uint32_t DAEMON_RPC_VERSION_ZMQ = DAEMON_RPC_VERSION_ZMQ_MINOR + (DAEMON_RPC_VERSION_ZMQ_MAJOR << 16); diff --git a/src/serialization/unordered_set.h b/src/rpc/fwd.h index 24aa09437..72537f5a5 100644 --- a/src/serialization/unordered_set.h +++ b/src/rpc/fwd.h @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2020, The Monero Project -// +// Copyright (c) 2019-2020, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL @@ -25,34 +25,13 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once -#include <set> - -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v); -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v); - -namespace serialization +namespace cryptonote { - namespace detail + namespace listener { - template <typename T> - void do_add(std::unordered_set<T> &c, T &&e) - { - c.insert(std::move(e)); - } + class zmq_pub; } } - -#include "serialization.h" - -template <template <bool> class Archive, class T> -bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> -bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); } - diff --git a/src/rpc/message.cpp b/src/rpc/message.cpp index e4f17cef8..f6c6b887d 100644 --- a/src/rpc/message.cpp +++ b/src/rpc/message.cpp @@ -65,8 +65,6 @@ const rapidjson::Value& get_method_field(const rapidjson::Value& src) void Message::toJson(rapidjson::Writer<epee::byte_stream>& dest) const { dest.StartObject(); - INSERT_INTO_JSON_OBJECT(dest, status, status); - INSERT_INTO_JSON_OBJECT(dest, error_details, error_details); INSERT_INTO_JSON_OBJECT(dest, rpc_version, DAEMON_RPC_VERSION_ZMQ); doToJson(dest); dest.EndObject(); @@ -74,14 +72,15 @@ void Message::toJson(rapidjson::Writer<epee::byte_stream>& dest) const void Message::fromJson(const rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, status, status); - GET_FROM_JSON_OBJECT(val, error_details, error_details); GET_FROM_JSON_OBJECT(val, rpc_version, rpc_version); } -FullMessage::FullMessage(const std::string& json_string, bool request) +FullMessage::FullMessage(std::string&& json_string, bool request) + : contents(std::move(json_string)), doc() { - doc.Parse(json_string.c_str()); + /* Insitu parsing does not copy data from `contents` to DOM, + accelerating string heavy content. */ + doc.ParseInsitu(std::addressof(contents[0])); if (doc.HasParseError() || !doc.IsObject()) { throw cryptonote::json::PARSE_FAIL(); diff --git a/src/rpc/message.h b/src/rpc/message.h index b858a5913..04bf1a111 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -72,9 +72,7 @@ namespace rpc public: ~FullMessage() { } - FullMessage(FullMessage&& rhs) noexcept : doc(std::move(rhs.doc)) { } - - FullMessage(const std::string& json_string, bool request=false); + FullMessage(std::string&& json_string, bool request=false); std::string getRequestType() const; @@ -91,10 +89,13 @@ namespace rpc private: FullMessage() = default; + FullMessage(const FullMessage&) = delete; + FullMessage& operator=(const FullMessage&) = delete; FullMessage(const std::string& request, Message* message); FullMessage(Message* message); + std::string contents; rapidjson::Document doc; }; diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 97dd0598b..9757fc462 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -55,7 +55,7 @@ class RpcHandler RpcHandler() { } virtual ~RpcHandler() { } - virtual epee::byte_slice handle(const std::string& request) = 0; + virtual epee::byte_slice handle(std::string&& request) = 0; static boost::optional<output_distribution_data> get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height); diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index edc8f0dda..176f11fa3 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -27,14 +27,12 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/archive/portable_binary_iarchive.hpp> -#include <boost/archive/portable_binary_oarchive.hpp> #include "cryptonote_config.h" #include "include_base_utils.h" #include "string_tools.h" #include "file_io_utils.h" #include "int-util.h" #include "common/util.h" -#include "serialization/crypto.h" #include "common/unordered_containers_boost_serialization.h" #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_basic/cryptonote_format_utils.h" @@ -296,14 +294,28 @@ namespace cryptonote data.open(state_file_path, std::ios_base::binary | std::ios_base::in); if (!data.fail()) { + bool loaded = false; try { - boost::archive::portable_binary_iarchive a(data); - a >> *this; + binary_archive<false> ar(data); + if (::serialization::serialize(ar, *this)) + if (::serialization::check_stream_state(ar)) + loaded = true; } - catch (const std::exception &e) + catch (...) {} + if (!loaded) { - MERROR("Failed to load RPC payments file: " << e.what()); + try + { + boost::archive::portable_binary_iarchive a(data); + a >> *this; + loaded = true; + } + catch (...) {} + } + if (!loaded) + { + MERROR("Failed to load RPC payments file"); m_client_info.clear(); } } @@ -344,8 +356,9 @@ namespace cryptonote MWARNING("Failed to save RPC payments to file " << state_file_path); return false; }; - boost::archive::portable_binary_oarchive a(data); - a << *this; + binary_archive<true> ar(data); + if (!::serialization::serialize(ar, *const_cast<rpc_payment*>(this))) + return false; return true; CATCH_ENTRY_L0("rpc_payment::store", false); } diff --git a/src/rpc/rpc_payment.h b/src/rpc/rpc_payment.h index dcd43f8d5..fdf1f953f 100644 --- a/src/rpc/rpc_payment.h +++ b/src/rpc/rpc_payment.h @@ -31,10 +31,17 @@ #include <string> #include <unordered_set> #include <unordered_map> +#include <map> #include <boost/thread/mutex.hpp> #include <boost/serialization/version.hpp> #include "cryptonote_basic/blobdatatype.h" #include "cryptonote_basic/cryptonote_basic.h" +#include <boost/serialization/list.hpp> +#include <boost/serialization/vector.hpp> +#include "serialization/crypto.h" +#include "serialization/string.h" +#include "serialization/pair.h" +#include "serialization/containers.h" namespace cryptonote { @@ -96,6 +103,33 @@ namespace cryptonote a & nonces_bad; a & nonces_dupe; } + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(block) + FIELD(previous_block) + FIELD(hashing_blob) + FIELD(previous_hashing_blob) + VARINT_FIELD(seed_height) + VARINT_FIELD(previous_seed_height) + FIELD(seed_hash) + FIELD(previous_seed_hash) + VARINT_FIELD(cookie) + FIELD(top) + FIELD(previous_top) + VARINT_FIELD(credits) + FIELD(payments) + FIELD(previous_payments) + FIELD(update_time) + FIELD(last_request_timestamp) + FIELD(block_template_update_time) + VARINT_FIELD(credits_total) + VARINT_FIELD(credits_used) + VARINT_FIELD(nonces_good) + VARINT_FIELD(nonces_stale) + VARINT_FIELD(nonces_bad) + VARINT_FIELD(nonces_dupe) + END_SERIALIZE() }; public: @@ -114,8 +148,8 @@ namespace cryptonote template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) { - a & m_client_info; - a & m_hashrate; + a & m_client_info.parent(); + a & m_hashrate.parent(); a & m_credits_total; a & m_credits_used; a & m_nonces_good; @@ -124,6 +158,18 @@ namespace cryptonote a & m_nonces_dupe; } + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(m_client_info) + FIELD(m_hashrate) + VARINT_FIELD(m_credits_total) + VARINT_FIELD(m_credits_used) + VARINT_FIELD(m_nonces_good) + VARINT_FIELD(m_nonces_stale) + VARINT_FIELD(m_nonces_bad) + VARINT_FIELD(m_nonces_dupe) + END_SERIALIZE() + bool load(std::string directory); bool store(const std::string &directory = std::string()) const; @@ -131,9 +177,9 @@ namespace cryptonote cryptonote::account_public_address m_address; uint64_t m_diff; uint64_t m_credits_per_hash_found; - std::unordered_map<crypto::public_key, client_info> m_client_info; + serializable_unordered_map<crypto::public_key, client_info> m_client_info; std::string m_directory; - std::map<uint64_t, uint64_t> m_hashrate; + serializable_map<uint64_t, uint64_t> m_hashrate; uint64_t m_credits_total; uint64_t m_credits_used; uint64_t m_nonces_good; @@ -143,6 +189,3 @@ namespace cryptonote mutable boost::mutex mutex; }; } - -BOOST_CLASS_VERSION(cryptonote::rpc_payment, 0); -BOOST_CLASS_VERSION(cryptonote::rpc_payment::client_info, 0); diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp new file mode 100644 index 000000000..0dffffac6 --- /dev/null +++ b/src/rpc/zmq_pub.cpp @@ -0,0 +1,478 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "zmq_pub.h" + +#include <algorithm> +#include <boost/range/adaptor/filtered.hpp> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/thread/locks.hpp> +#include <cassert> +#include <cstdint> +#include <cstring> +#include <rapidjson/document.h> +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> +#include <stdexcept> +#include <string> +#include <utility> + +#include "common/expect.h" +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_format_utils.h" +#include "cryptonote_basic/events.h" +#include "misc_log_ex.h" +#include "serialization/json_object.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.zmq" + +namespace +{ + constexpr const char txpool_signal[] = "tx_signal"; + + using chain_writer = void(epee::byte_stream&, std::uint64_t, epee::span<const cryptonote::block>); + using txpool_writer = void(epee::byte_stream&, epee::span<const cryptonote::txpool_event>); + + template<typename F> + struct context + { + char const* const name; + F* generate_pub; + }; + + template<typename T> + bool operator<(const context<T>& lhs, const context<T>& rhs) noexcept + { + return std::strcmp(lhs.name, rhs.name) < 0; + } + + template<typename T> + bool operator<(const context<T>& lhs, const boost::string_ref rhs) noexcept + { + return lhs.name < rhs; + } + + struct is_valid + { + bool operator()(const cryptonote::txpool_event& event) const noexcept + { + return event.res; + } + }; + + template<typename T, std::size_t N> + void verify_sorted(const std::array<context<T>, N>& elems, const char* name) + { + auto unsorted = std::is_sorted_until(elems.begin(), elems.end()); + if (unsorted != elems.end()) + throw std::logic_error{name + std::string{" array is not properly sorted, see: "} + unsorted->name}; + } + + void write_header(epee::byte_stream& buf, const boost::string_ref name) + { + buf.write(name.data(), name.size()); + buf.put(':'); + } + + //! \return `name:...` where `...` is JSON and `name` is directly copied (no quotes - not JSON). + template<typename T> + void json_pub(epee::byte_stream& buf, const T value) + { + rapidjson::Writer<epee::byte_stream> dest{buf}; + using cryptonote::json::toJsonValue; + toJsonValue(dest, value); + } + + //! Object for "minimal" block serialization + struct minimal_chain + { + const std::uint64_t height; + const epee::span<const cryptonote::block> blocks; + }; + + //! Object for "minimal" tx serialization + struct minimal_txpool + { + const cryptonote::transaction& tx; + }; + + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain self) + { + namespace adapt = boost::adaptors; + + const auto to_block_id = [](const cryptonote::block& bl) + { + crypto::hash id; + if (!get_block_hash(bl, id)) + MERROR("ZMQ/Pub failure: get_block_hash"); + return id; + }; + + assert(!self.blocks.empty()); // checked in zmq_pub::send_chain_main + + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, first_height, self.height); + INSERT_INTO_JSON_OBJECT(dest, first_prev_id, self.blocks[0].prev_id); + INSERT_INTO_JSON_OBJECT(dest, ids, (self.blocks | adapt::transformed(to_block_id))); + dest.EndObject(); + } + + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool self) + { + crypto::hash id{}; + std::size_t blob_size = 0; + if (!get_transaction_hash(self.tx, id, blob_size)) + { + MERROR("ZMQ/Pub failure: get_transaction_hash"); + return; + } + + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, id, id); + INSERT_INTO_JSON_OBJECT(dest, blob_size, blob_size); + dest.EndObject(); + } + + void json_full_chain(epee::byte_stream& buf, const std::uint64_t height, const epee::span<const cryptonote::block> blocks) + { + json_pub(buf, blocks); + } + + void json_minimal_chain(epee::byte_stream& buf, const std::uint64_t height, const epee::span<const cryptonote::block> blocks) + { + json_pub(buf, minimal_chain{height, blocks}); + } + + // boost::adaptors are in place "views" - no copy/move takes place + // moving transactions (via sort, etc.), is expensive! + + void json_full_txpool(epee::byte_stream& buf, epee::span<const cryptonote::txpool_event> txes) + { + namespace adapt = boost::adaptors; + const auto to_full_tx = [](const cryptonote::txpool_event& event) + { + return event.tx; + }; + json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_full_tx))); + } + + void json_minimal_txpool(epee::byte_stream& buf, epee::span<const cryptonote::txpool_event> txes) + { + namespace adapt = boost::adaptors; + const auto to_minimal_tx = [](const cryptonote::txpool_event& event) + { + return minimal_txpool{event.tx}; + }; + json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_minimal_tx))); + } + + constexpr const std::array<context<chain_writer>, 2> chain_contexts = + {{ + {u8"json-full-chain_main", json_full_chain}, + {u8"json-minimal-chain_main", json_minimal_chain} + }}; + + constexpr const std::array<context<txpool_writer>, 2> txpool_contexts = + {{ + {u8"json-full-txpool_add", json_full_txpool}, + {u8"json-minimal-txpool_add", json_minimal_txpool} + }}; + + template<typename T, std::size_t N> + epee::span<const context<T>> get_range(const std::array<context<T>, N>& contexts, const boost::string_ref value) + { + const auto not_prefix = [](const boost::string_ref lhs, const context<T>& rhs) + { + return !(boost::string_ref{rhs.name}.starts_with(lhs)); + }; + + const auto lower = std::lower_bound(contexts.begin(), contexts.end(), value); + const auto upper = std::upper_bound(lower, contexts.end(), value, not_prefix); + return {lower, std::size_t(upper - lower)}; + } + + template<std::size_t N, typename T> + void add_subscriptions(std::array<std::size_t, N>& subs, const epee::span<const context<T>> range, context<T> const* const first) + { + assert(range.size() <= N); + assert(range.begin() - first <= N - range.size()); + + for (const auto& ctx : range) + { + const std::size_t i = std::addressof(ctx) - first; + subs[i] = std::min(std::numeric_limits<std::size_t>::max() - 1, subs[i]) + 1; + } + } + + template<std::size_t N, typename T> + void remove_subscriptions(std::array<std::size_t, N>& subs, const epee::span<const context<T>> range, context<T> const* const first) + { + assert(range.size() <= N); + assert(range.begin() - first <= N - range.size()); + + for (const auto& ctx : range) + { + const std::size_t i = std::addressof(ctx) - first; + subs[i] = std::max(std::size_t(1), subs[i]) - 1; + } + } + + template<std::size_t N, typename T, typename... U> + std::array<epee::byte_slice, N> make_pubs(const std::array<std::size_t, N>& subs, const std::array<context<T>, N>& contexts, U&&... args) + { + epee::byte_stream buf{}; + + std::size_t last_offset = 0; + std::array<std::size_t, N> offsets{{}}; + for (std::size_t i = 0; i < N; ++i) + { + if (subs[i]) + { + write_header(buf, contexts[i].name); + contexts[i].generate_pub(buf, std::forward<U>(args)...); + offsets[i] = buf.size() - last_offset; + last_offset = buf.size(); + } + } + + epee::byte_slice bytes{std::move(buf)}; + std::array<epee::byte_slice, N> out; + for (std::size_t i = 0; i < N; ++i) + out[i] = bytes.take_slice(offsets[i]); + + return out; + } + + template<std::size_t N> + std::size_t send_messages(void* const socket, std::array<epee::byte_slice, N>& messages) + { + std::size_t count = 0; + for (epee::byte_slice& message : messages) + { + if (!message.empty()) + { + const expect<void> sent = net::zmq::send(std::move(message), socket, ZMQ_DONTWAIT); + if (!sent) + MERROR("Failed to send ZMQ/Pub message: " << sent.error().message()); + else + ++count; + } + } + return count; + } + + expect<bool> relay_block_pub(void* const relay, void* const pub) noexcept + { + zmq_msg_t msg; + zmq_msg_init(std::addressof(msg)); + MONERO_CHECK(net::zmq::retry_op(zmq_msg_recv, std::addressof(msg), relay, ZMQ_DONTWAIT)); + + const boost::string_ref payload{ + reinterpret_cast<const char*>(zmq_msg_data(std::addressof(msg))), + zmq_msg_size(std::addressof(msg)) + }; + + if (payload == txpool_signal) + { + zmq_msg_close(std::addressof(msg)); + return false; + } + + // forward block messages (serialized on P2P thread for now) + const expect<void> sent = net::zmq::retry_op(zmq_msg_send, std::addressof(msg), pub, ZMQ_DONTWAIT); + if (!sent) + { + zmq_msg_close(std::addressof(msg)); + return sent.error(); + } + return true; + } +} // anonymous + +namespace cryptonote { namespace listener +{ + +zmq_pub::zmq_pub(void* context) + : relay_(), + chain_subs_{{0}}, + txpool_subs_{{0}}, + sync_() +{ + if (!context) + throw std::logic_error{"ZMQ context cannot be NULL"}; + + verify_sorted(chain_contexts, "chain_contexts"); + verify_sorted(txpool_contexts, "txpool_contexts"); + + relay_.reset(zmq_socket(context, ZMQ_PAIR)); + if (!relay_) + MONERO_ZMQ_THROW("Failed to create relay socket"); + if (zmq_connect(relay_.get(), relay_endpoint()) != 0) + MONERO_ZMQ_THROW("Failed to connect relay socket"); +} + +zmq_pub::~zmq_pub() +{} + +bool zmq_pub::sub_request(boost::string_ref message) +{ + if (!message.empty()) + { + const char tag = message[0]; + message.remove_prefix(1); + + const auto chain_range = get_range(chain_contexts, message); + const auto txpool_range = get_range(txpool_contexts, message); + + if (!chain_range.empty() || !txpool_range.empty()) + { + MDEBUG("Client " << (tag ? "subscribed" : "unsubscribed") << " to " << + chain_range.size() << " chain topic(s) and " << txpool_range.size() << " txpool topic(s)"); + + const boost::lock_guard<boost::mutex> lock{sync_}; + switch (tag) + { + case 0: + remove_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + remove_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); + return true; + case 1: + add_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + add_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); + return true; + default: + break; + } + } + } + MERROR("Invalid ZMQ/Sub message"); + return false; +} + +bool zmq_pub::relay_to_pub(void* const relay, void* const pub) +{ + const expect<bool> relayed = relay_block_pub(relay, pub); + if (!relayed) + { + MERROR("Error relaying ZMQ/Pub: " << relayed.error().message()); + return false; + } + + if (!*relayed) + { + std::array<std::size_t, 2> subs; + std::vector<cryptonote::txpool_event> events; + { + const boost::lock_guard<boost::mutex> lock{sync_}; + if (txes_.empty()) + return false; + + subs = txpool_subs_; + events = std::move(txes_.front()); + txes_.pop_front(); + } + auto messages = make_pubs(subs, txpool_contexts, epee::to_span(events)); + send_messages(pub, messages); + MDEBUG("Sent txpool ZMQ/Pub"); + } + else + MDEBUG("Sent chain_main ZMQ/Pub"); + + return true; +} + +std::size_t zmq_pub::send_chain_main(const std::uint64_t height, const epee::span<const cryptonote::block> blocks) +{ + if (blocks.empty()) + return 0; + + /* Block format only sends one block at a time - multiple block notifications + are less common and only occur on rollbacks. */ + + boost::unique_lock<boost::mutex> guard{sync_}; + + const auto subs_copy = chain_subs_; + guard.unlock(); + + for (const std::size_t sub : subs_copy) + { + if (sub) + { + /* cryptonote_core/blockchain.cpp cannot "give" us the block like core + does for txpool events. Since copying the block is expensive anyway, + serialization is done right here on the p2p thread (for now). */ + + auto messages = make_pubs(subs_copy, chain_contexts, height, blocks); + guard.lock(); + return send_messages(relay_.get(), messages); + } + } + return 0; +} + +std::size_t zmq_pub::send_txpool_add(std::vector<txpool_event> txes) +{ + if (txes.empty()) + return 0; + + const boost::lock_guard<boost::mutex> lock{sync_}; + for (const std::size_t sub : txpool_subs_) + { + if (sub) + { + const expect<void> sent = net::zmq::retry_op(zmq_send_const, relay_.get(), txpool_signal, sizeof(txpool_signal) - 1, ZMQ_DONTWAIT); + if (sent) + txes_.emplace_back(std::move(txes)); + else + MERROR("ZMQ/Pub failure, relay queue error: " << sent.error().message()); + return bool(sent); + } + } + return 0; +} + +void zmq_pub::chain_main::operator()(const std::uint64_t height, epee::span<const cryptonote::block> blocks) const +{ + const std::shared_ptr<zmq_pub> self = self_.lock(); + if (self) + self->send_chain_main(height, blocks); + else + MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); +} + +void zmq_pub::txpool_add::operator()(std::vector<cryptonote::txpool_event> txes) const +{ + const std::shared_ptr<zmq_pub> self = self_.lock(); + if (self) + self->send_txpool_add(std::move(txes)); + else + MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); +} + +}} diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h new file mode 100644 index 000000000..02e6b8103 --- /dev/null +++ b/src/rpc/zmq_pub.h @@ -0,0 +1,110 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. + // +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <array> +#include <boost/thread/mutex.hpp> +#include <boost/utility/string_ref.hpp> +#include <cstdint> +#include <deque> +#include <memory> +#include <vector> + +#include "cryptonote_basic/fwd.h" +#include "net/zmq.h" +#include "span.h" + +namespace cryptonote { namespace listener +{ +/*! \brief Sends ZMQ PUB messages on cryptonote events + + Clients must ensure that all transaction(s) are notified before any blocks + they are contained in, and must ensure that each block is notified in chain + order. An external lock **must** be held by clients during the entire + txpool check and notification sequence and (a possibly second) lock is held + during the entire block check and notification sequence. Otherwise, events + could be sent in a different order than processed. */ +class zmq_pub +{ + /* Each socket has its own internal queue. So we can only use one socket, else + the messages being published are not guaranteed to be in the same order + pushed. */ + + net::zmq::socket relay_; + std::deque<std::vector<txpool_event>> txes_; + std::array<std::size_t, 2> chain_subs_; + std::array<std::size_t, 2> txpool_subs_; + boost::mutex sync_; //!< Synchronizes counts in `*_subs_` arrays. + + public: + //! \return Name of ZMQ_PAIR endpoint for pub notifications + static constexpr const char* relay_endpoint() noexcept { return "inproc://pub_relay"; } + + explicit zmq_pub(void* context); + + zmq_pub(const zmq_pub&) = delete; + zmq_pub(zmq_pub&&) = delete; + + ~zmq_pub(); + + zmq_pub& operator=(const zmq_pub&) = delete; + zmq_pub& operator=(zmq_pub&&) = delete; + + //! Process a client subscription request (from XPUB sockets). Thread-safe. + bool sub_request(const boost::string_ref message); + + /*! Forward ZMQ messages sent to `relay` via `send_chain_main` or + `send_txpool_add` to `pub`. Used by `ZmqServer`. */ + bool relay_to_pub(void* relay, void* pub); + + /*! Send a `ZMQ_PUB` notification for a change to the main chain. + Thread-safe. + \return Number of ZMQ messages sent to relay. */ + std::size_t send_chain_main(std::uint64_t height, epee::span<const cryptonote::block> blocks); + + /*! Send a `ZMQ_PUB` notification for new tx(es) being added to the local + pool. Thread-safe. + \return Number of ZMQ messages sent to relay. */ + std::size_t send_txpool_add(std::vector<cryptonote::txpool_event> txes); + + //! Callable for `send_chain_main` with weak ownership to `zmq_pub` object. + struct chain_main + { + std::weak_ptr<zmq_pub> self_; + void operator()(std::uint64_t height, epee::span<const cryptonote::block> blocks) const; + }; + + //! Callable for `send_txpool_add` with weak ownership to `zmq_pub` object. + struct txpool_add + { + std::weak_ptr<zmq_pub> self_; + void operator()(std::vector<cryptonote::txpool_event> txes) const; + }; + }; +}} diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 1a9f49c01..4028df96a 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -29,10 +29,16 @@ #include "zmq_server.h" #include <chrono> -#include <cstdint> +#include <cstring> +#include <utility> +#include <stdexcept> #include <system_error> #include "byte_slice.h" +#include "rpc/zmq_pub.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.zmq" namespace cryptonote { @@ -42,14 +48,57 @@ namespace constexpr const int num_zmq_threads = 1; constexpr const std::int64_t max_message_size = 10 * 1024 * 1024; // 10 MiB constexpr const std::chrono::seconds linger_timeout{2}; // wait period for pending out messages -} + + net::zmq::socket init_socket(void* context, int type, epee::span<const std::string> addresses) + { + if (context == nullptr) + throw std::logic_error{"NULL context provided"}; + + net::zmq::socket out{}; + out.reset(zmq_socket(context, type)); + if (!out) + { + MONERO_LOG_ZMQ_ERROR("Failed to create ZMQ socket"); + return nullptr; + } + + if (zmq_setsockopt(out.get(), ZMQ_MAXMSGSIZE, std::addressof(max_message_size), sizeof(max_message_size)) != 0) + { + MONERO_LOG_ZMQ_ERROR("Failed to set maximum incoming message size"); + return nullptr; + } + + static constexpr const int linger_value = std::chrono::milliseconds{linger_timeout}.count(); + if (zmq_setsockopt(out.get(), ZMQ_LINGER, std::addressof(linger_value), sizeof(linger_value)) != 0) + { + MONERO_LOG_ZMQ_ERROR("Failed to set linger timeout"); + return nullptr; + } + + for (const std::string& address : addresses) + { + if (zmq_bind(out.get(), address.c_str()) < 0) + { + MONERO_LOG_ZMQ_ERROR("ZMQ bind failed"); + return nullptr; + } + MINFO("ZMQ now listening at " << address); + } + + return out; + } +} // anonymous namespace rpc { ZmqServer::ZmqServer(RpcHandler& h) : handler(h), - context(zmq_init(num_zmq_threads)) + context(zmq_init(num_zmq_threads)), + rep_socket(nullptr), + pub_socket(nullptr), + relay_socket(nullptr), + shared_state(nullptr) { if (!context) MONERO_ZMQ_THROW("Unable to create ZMQ context"); @@ -64,22 +113,59 @@ void ZmqServer::serve() try { // socket must close before `zmq_term` will exit. - const net::zmq::socket socket = std::move(rep_socket); - if (!socket) + const net::zmq::socket rep = std::move(rep_socket); + const net::zmq::socket pub = std::move(pub_socket); + const net::zmq::socket relay = std::move(relay_socket); + const std::shared_ptr<listener::zmq_pub> state = std::move(shared_state); + + const unsigned init_count = unsigned(bool(pub)) + bool(relay) + bool(state); + if (!rep || (init_count && init_count != 3)) { - MERROR("ZMQ RPC server reply socket is null"); + MERROR("ZMQ RPC server socket is null"); return; } + MINFO("ZMQ Server started"); + + const int read_flags = pub ? ZMQ_DONTWAIT : 0; + std::array<zmq_pollitem_t, 3> sockets = + {{ + {relay.get(), 0, ZMQ_POLLIN, 0}, + {pub.get(), 0, ZMQ_POLLIN, 0}, + {rep.get(), 0, ZMQ_POLLIN, 0} + }}; + + /* This uses XPUB to watch for subscribers, to reduce CPU cycles for + serialization when the data will be dropped. This is important for block + serialization, which is done on the p2p threads currently (see + zmq_pub.cpp). + + XPUB sockets are not thread-safe, so the p2p thread cannot write into + the socket while we read here for subscribers. A ZMQ_PAIR socket is + used for inproc notification. No data is every copied to kernel, it is + all userspace messaging. */ + while (1) { - const std::string message = MONERO_UNWRAP(net::zmq::receive(socket.get())); - MDEBUG("Received RPC request: \"" << message << "\""); - epee::byte_slice response = handler.handle(message); + if (pub) + MONERO_UNWRAP(net::zmq::retry_op(zmq_poll, sockets.data(), sockets.size(), -1)); - const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; - MDEBUG("Sending RPC reply: \"" << response_view << "\""); - MONERO_UNWRAP(net::zmq::send(std::move(response), socket.get())); + if (sockets[0].revents) + state->relay_to_pub(relay.get(), pub.get()); + + if (sockets[1].revents) + state->sub_request(MONERO_UNWRAP(net::zmq::receive(pub.get(), ZMQ_DONTWAIT))); + + if (!pub || sockets[2].revents) + { + std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags)); + MDEBUG("Received RPC request: \"" << message << "\""); + epee::byte_slice response = handler.handle(std::move(message)); + + const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; + MDEBUG("Sending RPC reply: \"" << response_view << "\""); + MONERO_UNWRAP(net::zmq::send(std::move(response), rep.get())); + } } } catch (const std::system_error& e) @@ -97,38 +183,12 @@ void ZmqServer::serve() } } -bool ZmqServer::addIPCSocket(const boost::string_ref address, const boost::string_ref port) -{ - MERROR("ZmqServer::addIPCSocket not yet implemented!"); - return false; -} - -bool ZmqServer::addTCPSocket(boost::string_ref address, boost::string_ref port) +void* ZmqServer::init_rpc(boost::string_ref address, boost::string_ref port) { if (!context) { MERROR("ZMQ RPC Server already shutdown"); - return false; - } - - rep_socket.reset(zmq_socket(context.get(), ZMQ_REP)); - if (!rep_socket) - { - MONERO_LOG_ZMQ_ERROR("ZMQ RPC Server socket create failed"); - return false; - } - - if (zmq_setsockopt(rep_socket.get(), ZMQ_MAXMSGSIZE, std::addressof(max_message_size), sizeof(max_message_size)) != 0) - { - MONERO_LOG_ZMQ_ERROR("Failed to set maximum incoming message size"); - return false; - } - - static constexpr const int linger_value = std::chrono::milliseconds{linger_timeout}.count(); - if (zmq_setsockopt(rep_socket.get(), ZMQ_LINGER, std::addressof(linger_value), sizeof(linger_value)) != 0) - { - MONERO_LOG_ZMQ_ERROR("Failed to set linger timeout"); - return false; + return nullptr; } if (address.empty()) @@ -141,12 +201,34 @@ bool ZmqServer::addTCPSocket(boost::string_ref address, boost::string_ref port) bind_address += ":"; bind_address.append(port.data(), port.size()); - if (zmq_bind(rep_socket.get(), bind_address.c_str()) < 0) + rep_socket = init_socket(context.get(), ZMQ_REP, {std::addressof(bind_address), 1}); + return bool(rep_socket) ? context.get() : nullptr; +} + +std::shared_ptr<listener::zmq_pub> ZmqServer::init_pub(epee::span<const std::string> addresses) +{ + try + { + shared_state = std::make_shared<listener::zmq_pub>(context.get()); + pub_socket = init_socket(context.get(), ZMQ_XPUB, addresses); + if (!pub_socket) + throw std::runtime_error{"Unable to initialize ZMQ_XPUB socket"}; + + const std::string relay_address[] = {listener::zmq_pub::relay_endpoint()}; + relay_socket = init_socket(context.get(), ZMQ_PAIR, relay_address); + if (!relay_socket) + throw std::runtime_error{"Unable to initialize ZMQ_PAIR relay"}; + } + catch (const std::runtime_error& e) { - MONERO_LOG_ZMQ_ERROR("ZMQ RPC Server bind failed"); - return false; + shared_state = nullptr; + pub_socket = nullptr; + relay_socket = nullptr; + MERROR("Failed to create ZMQ/Pub listener: " << e.what()); + return nullptr; } - return true; + + return shared_state; } void ZmqServer::run() @@ -163,7 +245,6 @@ void ZmqServer::stop() run_thread.join(); } - } // namespace cryptonote } // namespace rpc diff --git a/src/rpc/zmq_server.h b/src/rpc/zmq_server.h index 1143db839..ddf44b411 100644 --- a/src/rpc/zmq_server.h +++ b/src/rpc/zmq_server.h @@ -30,10 +30,16 @@ #include <boost/thread/thread.hpp> #include <boost/utility/string_ref.hpp> +#include <cstdint> +#include <memory> +#include <string> #include "common/command_line.h" +#include "cryptonote_basic/fwd.h" #include "net/zmq.h" -#include "rpc_handler.h" +#include "rpc/fwd.h" +#include "rpc/rpc_handler.h" +#include "span.h" namespace cryptonote { @@ -41,7 +47,7 @@ namespace cryptonote namespace rpc { -class ZmqServer +class ZmqServer final { public: @@ -49,12 +55,13 @@ class ZmqServer ~ZmqServer(); - static void init_options(boost::program_options::options_description& desc); - void serve(); - bool addIPCSocket(boost::string_ref address, boost::string_ref port); - bool addTCPSocket(boost::string_ref address, boost::string_ref port); + //! \return ZMQ context on success, `nullptr` on failure + void* init_rpc(boost::string_ref address, boost::string_ref port); + + //! \return `nullptr` on errors. + std::shared_ptr<listener::zmq_pub> init_pub(epee::span<const std::string> addresses); void run(); void stop(); @@ -67,9 +74,11 @@ class ZmqServer boost::thread run_thread; net::zmq::socket rep_socket; + net::zmq::socket pub_socket; + net::zmq::socket relay_socket; + std::shared_ptr<listener::zmq_pub> shared_state; }; - } // namespace cryptonote } // namespace rpc diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt index a2e7c353e..34e274b6c 100644 --- a/src/serialization/CMakeLists.txt +++ b/src/serialization/CMakeLists.txt @@ -42,6 +42,7 @@ monero_add_library(serialization ${serialization_private_headers}) target_link_libraries(serialization LINK_PRIVATE + cryptonote_basic cryptonote_core cryptonote_protocol epee diff --git a/src/serialization/container.h b/src/serialization/container.h index 4bf47ecfa..d5e75bb4f 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -28,10 +28,6 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#pragma once - -#include "serialization.h" - namespace serialization { namespace detail @@ -103,7 +99,7 @@ bool do_serialize_container(Archive<true> &ar, C &v) return false; if (i != v.begin()) ar.delimit_array(); - if(!::serialization::detail::serialize_container_element(ar, const_cast<typename C::value_type&>(*i))) + if(!::serialization::detail::serialize_container_element(ar, (typename C::value_type&)*i)) return false; if (!ar.stream().good()) return false; diff --git a/src/serialization/containers.h b/src/serialization/containers.h new file mode 100644 index 000000000..bc4a89527 --- /dev/null +++ b/src/serialization/containers.h @@ -0,0 +1,128 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include <vector> +#include <deque> +#include <unordered_map> +#include <map> +#include <unordered_set> +#include <set> +#include "serialization.h" + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::vector<T> &v); +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::vector<T> &v); + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::deque<T> &v); +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::deque<T> &v); + +template<typename K, typename V> +class serializable_unordered_map: public std::unordered_map<K, V> +{ +public: + typedef typename std::pair<K, V> value_type; + typename std::unordered_map<K, V> &parent() { return *this; } +}; + +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_map<K, V> &v); +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_map<K, V> &v); + +template<typename K, typename V> +class serializable_map: public std::map<K, V> +{ +public: + typedef typename std::pair<K, V> value_type; + typename std::map<K, V> &parent() { return *this; } +}; + +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_map<K, V> &v); +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_map<K, V> &v); + +template<typename K, typename V> +class serializable_unordered_multimap: public std::unordered_multimap<K, V> +{ +public: + typedef typename std::pair<K, V> value_type; + typename std::unordered_multimap<K, V> &parent() { return *this; } +}; + +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_multimap<K, V> &v); +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_multimap<K, V> &v); + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v); +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v); + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::set<T> &v); +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::set<T> &v); + +namespace serialization +{ + namespace detail + { + template <typename T> void do_reserve(std::vector<T> &c, size_t N) { c.reserve(N); } + template <typename T> void do_add(std::vector<T> &c, T &&e) { c.emplace_back(std::forward<T>(e)); } + + template <typename T> void do_add(std::deque<T> &c, T &&e) { c.emplace_back(std::forward<T>(e)); } + + template <typename K, typename V> void do_add(serializable_unordered_map<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); } + + template <typename K, typename V> void do_add(serializable_map<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); } + + template <typename K, typename V> void do_add(serializable_unordered_multimap<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); } + + template <typename T> void do_add(std::unordered_set<T> &c, T &&e) { c.insert(std::forward<T>(e)); } + + template <typename T> void do_add(std::set<T> &c, T &&e) { c.insert(std::forward<T>(e)); } + } +} + +#include "container.h" + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); } + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); } + +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_map<K, V> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_map<K, V> &v) { return do_serialize_container(ar, v); } + +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_map<K, V> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_map<K, V> &v) { return do_serialize_container(ar, v); } + +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_multimap<K, V> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_multimap<K, V> &v) { return do_serialize_container(ar, v); } + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); } + +template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::set<T> &v) { return do_serialize_container(ar, v); } +template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::set<T> &v) { return do_serialize_container(ar, v); } diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 5c042aa7b..7c48cf6c3 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -33,6 +33,8 @@ #include <limits> #include <type_traits> +#include "cryptonote_basic/cryptonote_basic_impl.h" + // drop macro from windows.h #ifdef GetObject #undef GetObject @@ -146,6 +148,26 @@ void fromJsonValue(const rapidjson::Value& val, std::string& str) str = val.GetString(); } +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const std::vector<std::uint8_t>& src) +{ + const std::string hex = epee::to_hex::string(epee::to_span(src)); + dest.String(hex.data(), hex.size()); +} + +void fromJsonValue(const rapidjson::Value& val, std::vector<std::uint8_t>& dest) +{ + if (!val.IsString()) + { + throw WRONG_TYPE("string"); + } + + dest.resize(val.GetStringLength() / 2); + if ((val.GetStringLength() % 2) != 0 || !epee::from_hex::to_buffer(epee::to_mut_span(dest), {val.GetString(), val.GetStringLength()})) + { + throw BAD_INPUT(); + } +} + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i) { dest.Bool(i); @@ -246,7 +268,10 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::t INSERT_INTO_JSON_OBJECT(dest, inputs, tx.vin); INSERT_INTO_JSON_OBJECT(dest, outputs, tx.vout); INSERT_INTO_JSON_OBJECT(dest, extra, tx.extra); - INSERT_INTO_JSON_OBJECT(dest, signatures, tx.signatures); + if (!tx.pruned) + { + INSERT_INTO_JSON_OBJECT(dest, signatures, tx.signatures); + } INSERT_INTO_JSON_OBJECT(dest, ringct, tx.rct_signatures); dest.EndObject(); @@ -265,8 +290,17 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx) GET_FROM_JSON_OBJECT(val, tx.vin, inputs); GET_FROM_JSON_OBJECT(val, tx.vout, outputs); GET_FROM_JSON_OBJECT(val, tx.extra, extra); - GET_FROM_JSON_OBJECT(val, tx.signatures, signatures); GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct); + + const auto& sigs = val.FindMember("signatures"); + if (sigs != val.MemberEnd()) + { + fromJsonValue(sigs->value, tx.signatures); + } + + const auto& rsig = tx.rct_signatures; + if (!cryptonote::is_coinbase(tx) && rsig.p.bulletproofs.empty() && rsig.p.rangeSigs.empty() && rsig.p.MGs.empty() && rsig.get_pseudo_outs().empty() && sigs == val.MemberEnd()) + tx.pruned = true; } void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b) @@ -1062,6 +1096,7 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& INSERT_INTO_JSON_OBJECT(dest, fee, sig.txnFee); // prunable + if (!sig.p.bulletproofs.empty() || !sig.p.rangeSigs.empty() || !sig.p.MGs.empty() || !sig.get_pseudo_outs().empty()) { dest.Key("prunable"); dest.StartObject(); @@ -1086,35 +1121,39 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig) throw WRONG_TYPE("json object"); } - std::vector<rct::key> commitments; - GET_FROM_JSON_OBJECT(val, sig.type, type); GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); - GET_FROM_JSON_OBJECT(val, commitments, commitments); + GET_FROM_JSON_OBJECT(val, sig.outPk, commitments); GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); // prunable + const auto prunable = val.FindMember("prunable"); + if (prunable != val.MemberEnd()) { - OBJECT_HAS_MEMBER_OR_THROW(val, "prunable"); - const auto& prunable = val["prunable"]; - - rct::keyV pseudo_outs; + rct::keyV pseudo_outs = std::move(sig.get_pseudo_outs()); - GET_FROM_JSON_OBJECT(prunable, sig.p.rangeSigs, range_proofs); - GET_FROM_JSON_OBJECT(prunable, sig.p.bulletproofs, bulletproofs); - GET_FROM_JSON_OBJECT(prunable, sig.p.MGs, mlsags); - GET_FROM_JSON_OBJECT(prunable, pseudo_outs, pseudo_outs); + GET_FROM_JSON_OBJECT(prunable->value, sig.p.rangeSigs, range_proofs); + GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs, bulletproofs); + GET_FROM_JSON_OBJECT(prunable->value, sig.p.MGs, mlsags); + GET_FROM_JSON_OBJECT(prunable->value, pseudo_outs, pseudo_outs); sig.get_pseudo_outs() = std::move(pseudo_outs); } - - sig.outPk.reserve(commitments.size()); - for (rct::key const& commitment : commitments) + else { - sig.outPk.push_back({{}, commitment}); + sig.p.rangeSigs.clear(); + sig.p.bulletproofs.clear(); + sig.p.MGs.clear(); + sig.get_pseudo_outs().clear(); } } +void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key) +{ + key.dest = {}; + fromJsonValue(val, key.mask); +} + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple) { dest.StartObject(); diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 2a9b63b08..de14c8911 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -32,6 +32,7 @@ #include <cstring> #include <rapidjson/document.h> #include <rapidjson/writer.h> +#include <vector> #include "byte_stream.h" #include "cryptonote_basic/cryptonote_basic.h" @@ -153,6 +154,9 @@ inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const std::s } void fromJsonValue(const rapidjson::Value& val, std::string& str); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const std::vector<std::uint8_t>&); +void fromJsonValue(const rapidjson::Value& src, std::vector<std::uint8_t>& i); + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i); void fromJsonValue(const rapidjson::Value& val, bool& b); @@ -277,6 +281,8 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i); void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig); +void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key); + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple); void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple); @@ -339,6 +345,7 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from auto itr = val.MemberBegin(); + map.clear(); while (itr != val.MemberEnd()) { typename Map::key_type k; @@ -353,25 +360,47 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from template <typename Vec> inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec) { + using value_type = typename Vec::value_type; + static_assert(!std::is_same<value_type, char>::value, "encoding an array of chars is faster as hex"); + static_assert(!std::is_same<value_type, unsigned char>::value, "encoding an array of unsigned char is faster as hex"); + dest.StartArray(); for (const auto& t : vec) toJsonValue(dest, t); - dest.EndArray(vec.size()); + dest.EndArray(); +} + +namespace traits +{ + template<typename T> + void reserve(const T&, std::size_t) + {} + + template<typename T> + void reserve(std::vector<T>& vec, const std::size_t count) + { + vec.reserve(count); + } } template <typename Vec> inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec) { + using value_type = typename Vec::value_type; + static_assert(!std::is_same<value_type, char>::value, "encoding a vector of chars is faster as hex"); + static_assert(!std::is_same<value_type, unsigned char>::value, "encoding a vector of unsigned char is faster as hex"); + if (!val.IsArray()) { throw WRONG_TYPE("json array"); } + vec.clear(); + traits::reserve(vec, val.Size()); for (rapidjson::SizeType i=0; i < val.Size(); i++) { - typename Vec::value_type v; - fromJsonValue(val[i], v); - vec.push_back(v); + vec.emplace_back(); + fromJsonValue(val[i], vec.back()); } } diff --git a/src/serialization/list.h b/src/serialization/list.h index 139269b40..411e7800d 100644 --- a/src/serialization/list.h +++ b/src/serialization/list.h @@ -44,7 +44,7 @@ namespace serialization template <typename T> void do_add(std::list<T> &c, T &&e) { - c.emplace_back(std::move(e)); + c.emplace_back(std::forward<T>(e)); } } } diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index d0f9ee0f7..3ebeed171 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -48,6 +48,7 @@ #include <string> #include <boost/type_traits/is_integral.hpp> #include <boost/type_traits/integral_constant.hpp> +#include <boost/mpl/bool.hpp> /*! \struct is_blob_type * @@ -278,6 +279,27 @@ inline bool do_serialize(Archive &ar, bool &v) if (!ar.stream().good()) return false; \ } while(0); +/*! \macro MAGIC_FIELD(m) + */ +#define MAGIC_FIELD(m) \ + std::string magic = m; \ + do { \ + ar.tag("magic"); \ + ar.serialize_blob((void*)magic.data(), magic.size()); \ + if (!ar.stream().good()) return false; \ + if (magic != m) return false; \ + } while(0); + +/*! \macro VERSION_FIELD(v) + */ +#define VERSION_FIELD(v) \ + uint32_t version = v; \ + do { \ + ar.tag("version"); \ + ar.serialize_varint(version); \ + if (!ar.stream().good()) return false; \ + } while(0); + namespace serialization { /*! \namespace detail diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index d36e5009b..02540a844 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -128,7 +128,7 @@ typedef cryptonote::simple_wallet sw; #define SCOPED_WALLET_UNLOCK() SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return true;) -#define PRINT_USAGE(usage_help_advanced) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help_advanced; +#define PRINT_USAGE(usage_help) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help; #define LONG_PAYMENT_ID_SUPPORT_CHECK() \ do { \ @@ -208,7 +208,7 @@ namespace const char* USAGE_ADDRESS_BOOK("address_book [(add (<address>|<integrated address>) [<description possibly with whitespaces>])|(delete <index>)]"); const char* USAGE_SET_VARIABLE("set <option> [<value>]"); const char* USAGE_GET_TX_KEY("get_tx_key <txid>"); - const char* USAGE_SET_TX_KEY("set_tx_key <txid> <tx_key>"); + const char* USAGE_SET_TX_KEY("set_tx_key <txid> <tx_key> [<subaddress>]"); const char* USAGE_CHECK_TX_KEY("check_tx_key <txid> <txkey> <address>"); const char* USAGE_GET_TX_PROOF("get_tx_proof <txid> <address> [<message>]"); const char* USAGE_CHECK_TX_PROOF("check_tx_proof <txid> <address> <signature_file> [<message>]"); @@ -279,8 +279,8 @@ namespace const char* USAGE_STOP_MINING_FOR_RPC("stop_mining_for_rpc"); const char* USAGE_SHOW_QR_CODE("show_qr_code [<subaddress_index>]"); const char* USAGE_VERSION("version"); - const char* USAGE_HELP_ADVANCED("help_advanced [<command>]"); - const char* USAGE_HELP("help"); + const char* USAGE_HELP("help [<command> | all]"); + const char* USAGE_APROPOS("apropos <keyword> [<keyword> ...]"); std::string input_line(const std::string& prompt, bool yesno = false) { @@ -2317,7 +2317,7 @@ bool simple_wallet::on_unknown_command(const std::vector<std::string> &args) { if (args[0] == "exit" || args[0] == "q") // backward compat return false; - fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help_advanced'")) % args.front(); + fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help'")) % args.front(); return true; } @@ -3100,39 +3100,63 @@ bool simple_wallet::set_export_format(const std::vector<std::string> &args/* = s return true; } -bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +bool simple_wallet::set_load_deprecated_formats(const std::vector<std::string> &args/* = std::vector<std::string()*/) { - message_writer() << ""; - message_writer() << tr("Commands:"); - message_writer() << ""; - message_writer() << tr("\"welcome\" - Read welcome message."); - message_writer() << tr("\"donate <amount>\" - Donate XMR to the development team."); - message_writer() << tr("\"balance\" - Show balance."); - message_writer() << tr("\"address new\" - Create new subaddress."); - message_writer() << tr("\"address all\" - Show all addresses."); - message_writer() << tr("\"transfer <address> <amount>\" - Send XMR to an address."); - message_writer() << tr("\"show_transfers [in|out|pending|failed|pool]\" - Show transactions."); - message_writer() << tr("\"sweep_all <address>\" - Send whole balance to another wallet."); - message_writer() << tr("\"seed\" - Show secret 25 words that can be used to recover this wallet."); - message_writer() << tr("\"refresh\" - Synchronize wallet with the Monero network."); - message_writer() << tr("\"status\" - Check current status of wallet."); - message_writer() << tr("\"version\" - Check software version."); - message_writer() << tr("\"help_advanced\" - Show list with more available commands."); - message_writer() << tr("\"save\" - Save wallet."); - message_writer() << tr("\"exit\" - Exit wallet."); - message_writer() << ""; + if (args.size() < 2) + { + fail_msg_writer() << tr("Value not specified"); + return true; + } + + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->load_deprecated_formats(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + + if (r) + message_writer() << tr("Warning: deprecated formats use boost serialization, which has buffer overflows and crashers. Only load deprecated formats from sources you trust."); + }); + } return true; } -bool simple_wallet::help_advanced(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) { + message_writer() << ""; + message_writer() << tr("Important commands:"); + message_writer() << ""; + message_writer() << tr("\"welcome\" - Show welcome message."); + message_writer() << tr("\"help all\" - Show the list of all available commands."); + message_writer() << tr("\"help <command>\" - Show a command's documentation."); + message_writer() << tr("\"apropos <keyword>\" - Show commands related to a keyword."); + message_writer() << ""; + message_writer() << tr("\"wallet_info\" - Show wallet main address and other info."); + message_writer() << tr("\"balance\" - Show balance."); + message_writer() << tr("\"address all\" - Show all addresses."); + message_writer() << tr("\"address new\" - Create new subaddress."); + message_writer() << tr("\"transfer <address> <amount>\" - Send XMR to an address."); + message_writer() << tr("\"show_transfers [in|out|pending|failed|pool]\" - Show transactions."); + message_writer() << tr("\"sweep_all <address>\" - Send whole balance to another wallet."); + message_writer() << tr("\"seed\" - Show secret 25 words that can be used to recover this wallet."); + message_writer() << tr("\"refresh\" - Synchronize wallet with the Monero network."); + message_writer() << tr("\"status\" - Check current status of wallet."); + message_writer() << tr("\"version\" - Check software version."); + message_writer() << tr("\"exit\" - Exit wallet."); + message_writer() << ""; + message_writer() << tr("\"donate <amount>\" - Donate XMR to the development team."); + message_writer() << ""; + } + else if ((args.size() == 1) && (args.front() == "all")) + { success_msg_writer() << get_commands_str(); } else if ((args.size() == 2) && (args.front() == "mms")) { - // Little hack to be able to do "help_advanced mms <subcommand>" + // Little hack to be able to do "help mms <subcommand>" std::vector<std::string> mms_args(1, args.front() + " " + args.back()); success_msg_writer() << get_command_usage(mms_args); } @@ -3143,6 +3167,33 @@ bool simple_wallet::help_advanced(const std::vector<std::string> &args/* = std:: return true; } +bool simple_wallet::apropos(const std::vector<std::string> &args) +{ + if (args.empty()) + { + PRINT_USAGE(USAGE_APROPOS); + return true; + } + const std::vector<std::string>& command_list = m_cmd_binder.get_command_list(args); + if (command_list.empty()) + { + fail_msg_writer() << tr("No commands found mentioning keyword(s)"); + return true; + } + + success_msg_writer() << ""; + for(auto const& command:command_list) + { + std::vector<std::string> cmd; + cmd.push_back(command); + std::pair<std::string, std::string> documentation = m_cmd_binder.get_documentation(cmd); + success_msg_writer() << " " << documentation.first; + } + success_msg_writer() << ""; + + return true; +} + simple_wallet::simple_wallet() : m_allow_mismatched_daemon_version(false) , m_refresh_progress_reporter(*this) @@ -3526,7 +3577,7 @@ simple_wallet::simple_wallet() "<subcommand> is one of:\n" " init, info, signer, list, next, sync, transfer, delete, send, receive, export, note, show, set, help\n" " send_signer_config, start_auto_config, stop_auto_config, auto_config, config_checksum\n" - "Get help about a subcommand with: help_advanced mms <subcommand>")); + "Get help about a subcommand with: help mms <subcommand>, or help mms <subcommand>")); m_cmd_binder.set_handler("mms init", boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1), tr(USAGE_MMS_INIT), @@ -3684,14 +3735,14 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::on_command, this, &simple_wallet::show_qr_code, _1), tr(USAGE_SHOW_QR_CODE), tr("Show address as QR code")); - m_cmd_binder.set_handler("help_advanced", - boost::bind(&simple_wallet::on_command, this, &simple_wallet::help_advanced, _1), - tr(USAGE_HELP_ADVANCED), - tr("Show the help section or the documentation about a <command>.")); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::on_command, this, &simple_wallet::help, _1), tr(USAGE_HELP), - tr("Show simplified list of available commands.")); + tr("Show the help section or the documentation about a <command>.")); + m_cmd_binder.set_handler("apropos", + boost::bind(&simple_wallet::on_command, this, &simple_wallet::apropos, _1), + tr(USAGE_APROPOS), + tr("Search all command descriptions for keyword(s)")); m_cmd_binder.set_unknown_command_handler(boost::bind(&simple_wallet::on_command, this, &simple_wallet::on_unknown_command, _1)); m_cmd_binder.set_empty_command_handler(boost::bind(&simple_wallet::on_empty_command, this)); m_cmd_binder.set_cancel_handler(boost::bind(&simple_wallet::on_cancelled_command, this)); @@ -3760,6 +3811,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "persistent-rpc-client-id = " << m_wallet->persistent_rpc_client_id(); success_msg_writer() << "auto-mine-for-rpc-payment-threshold = " << m_wallet->auto_mine_for_rpc_payment_threshold(); success_msg_writer() << "credits-target = " << m_wallet->credits_target(); + success_msg_writer() << "load-deprecated-formats = " << m_wallet->load_deprecated_formats(); return true; } else @@ -3821,6 +3873,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("setup-background-mining", set_setup_background_mining, tr("1/yes or 0/no")); CHECK_SIMPLE_VARIABLE("device-name", set_device_name, tr("<device_name[:device_spec]>")); CHECK_SIMPLE_VARIABLE("export-format", set_export_format, tr("\"binary\" or \"ascii\"")); + CHECK_SIMPLE_VARIABLE("load-deprecated-formats", set_load_deprecated_formats, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("persistent-rpc-client-id", set_persistent_rpc_client_id, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("auto-mine-for-rpc-payment-threshold", set_auto_mine_for_rpc_payment_threshold, tr("floating point >= 0")); CHECK_SIMPLE_VARIABLE("credits-target", set_credits_target, tr("unsigned integer")); @@ -4844,8 +4897,8 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr tr("Your wallet has been generated!\n" "To start synchronizing with the daemon, use the \"refresh\" command.\n" "Use the \"help\" command to see a simplified list of available commands.\n" - "Use the \"help_advanced\" command to see an advanced list of available commands.\n" - "Use \"help_advanced <command>\" to see a command's documentation.\n" + "Use \"help all\" command to see the list of all available commands.\n" + "Use \"help <command>\" to see a command's documentation.\n" "Always use the \"exit\" command when closing monero-wallet-cli to save \n" "your current session's state. Otherwise, you might need to synchronize \n" "your wallet again (your wallet keys are NOT at risk in any case).\n") @@ -5105,8 +5158,8 @@ boost::optional<epee::wipeable_string> simple_wallet::open_wallet(const boost::p success_msg_writer() << "**********************************************************************\n" << tr("Use the \"help\" command to see a simplified list of available commands.\n") << - tr("Use the \"help_advanced\" command to see an advanced list of available commands.\n") << - tr("Use \"help_advanced <command>\" to see a command's documentation.\n") << + tr("Use \"help all\" to see the list of all available commands.\n") << + tr("Use \"help <command>\" to see a command's documentation.\n") << "**********************************************************************"; return password; } @@ -6321,7 +6374,7 @@ void simple_wallet::check_for_inactivity_lock(bool user) m_in_command = true; if (!user) { - const std::string speech = tr("I locked your Monero wallet to protect you while you were away\nsee \"help_advanced set\" to configure/disable"); + const std::string speech = tr("I locked your Monero wallet to protect you while you were away\nsee \"help set\" to configure/disable"); std::vector<std::pair<std::string, size_t>> lines = tools::split_string_by_width(speech, 45); size_t max_len = 0; @@ -7439,6 +7492,32 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + std::vector<cryptonote::address_parse_info> dsts_info; + dsts_info.push_back(info); + + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return true; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + success_msg_writer(true) << tr("Money successfully sent, transaction: ") << get_transaction_hash(ptx_vector[0].tx); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); @@ -7884,11 +7963,27 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) { std::vector<std::string> local_args = args_; - if(local_args.size() != 2) { + if(local_args.size() != 2 && local_args.size() != 3) { PRINT_USAGE(USAGE_SET_TX_KEY); return true; } + boost::optional<cryptonote::account_public_address> single_destination_subaddress; + if (local_args.size() > 1) + { + cryptonote::address_parse_info info; + if (cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args.back(), oa_prompter)) + { + if (!info.is_subaddress) + { + fail_msg_writer() << tr("Last argument is an address, but not a subaddress"); + return true; + } + single_destination_subaddress = info.address; + local_args.pop_back(); + } + } + crypto::hash txid; if (!epee::string_tools::hex_to_pod(local_args[0], txid)) { @@ -7928,12 +8023,14 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) try { - m_wallet->set_tx_key(txid, tx_key, additional_tx_keys); + m_wallet->set_tx_key(txid, tx_key, additional_tx_keys, single_destination_subaddress); success_msg_writer() << tr("Tx key successfully stored."); } catch (const std::exception &e) { fail_msg_writer() << tr("Failed to store tx key: ") << e.what(); + if (!single_destination_subaddress) + fail_msg_writer() << tr("It could be because the transfer was to a subaddress. If this is the case, pass the subaddress last"); } return true; } @@ -11206,7 +11303,7 @@ void simple_wallet::mms_help(const std::vector<std::string> &args) { if (args.size() > 1) { - fail_msg_writer() << tr("Usage: help_advanced mms [<subcommand>]"); + fail_msg_writer() << tr("Usage: help mms [<subcommand>]"); return; } std::vector<std::string> help_args; diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 5154ff1ef..5846fe056 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -151,11 +151,12 @@ namespace cryptonote bool set_setup_background_mining(const std::vector<std::string> &args = std::vector<std::string>()); bool set_device_name(const std::vector<std::string> &args = std::vector<std::string>()); bool set_export_format(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_load_deprecated_formats(const std::vector<std::string> &args = std::vector<std::string>()); bool set_persistent_rpc_client_id(const std::vector<std::string> &args = std::vector<std::string>()); bool set_auto_mine_for_rpc_payment_threshold(const std::vector<std::string> &args = std::vector<std::string>()); bool set_credits_target(const std::vector<std::string> &args = std::vector<std::string>()); - bool help_advanced(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); + bool apropos(const std::vector<std::string> &args); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); bool set_daemon(const std::vector<std::string> &args); diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 66e248427..73ab88a9f 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -938,13 +938,13 @@ string WalletImpl::keysFilename() const return m_wallet->get_keys_file(); } -bool WalletImpl::init(const std::string &daemon_address, uint64_t upper_transaction_size_limit, const std::string &daemon_username, const std::string &daemon_password, bool use_ssl, bool lightWallet) +bool WalletImpl::init(const std::string &daemon_address, uint64_t upper_transaction_size_limit, const std::string &daemon_username, const std::string &daemon_password, bool use_ssl, bool lightWallet, const std::string &proxy_address) { clearStatus(); m_wallet->set_light_wallet(lightWallet); if(daemon_username != "") m_daemon_login.emplace(daemon_username, daemon_password); - return doInit(daemon_address, upper_transaction_size_limit, use_ssl); + return doInit(daemon_address, proxy_address, upper_transaction_size_limit, use_ssl); } bool WalletImpl::lightWalletLogin(bool &isNewWallet) const @@ -2088,6 +2088,11 @@ bool WalletImpl::trustedDaemon() const return m_wallet->is_trusted_daemon(); } +bool WalletImpl::setProxy(const std::string &address) +{ + return m_wallet->set_proxy(address); +} + bool WalletImpl::watchOnly() const { return m_wallet->watch_only(); @@ -2241,9 +2246,9 @@ void WalletImpl::pendingTxPostProcess(PendingTransactionImpl * pending) pending->m_pending_tx = exported_txs.ptx; } -bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) +bool WalletImpl::doInit(const string &daemon_address, const std::string &proxy_address, uint64_t upper_transaction_size_limit, bool ssl) { - if (!m_wallet->init(daemon_address, m_daemon_login, boost::asio::ip::tcp::endpoint{}, upper_transaction_size_limit)) + if (!m_wallet->init(daemon_address, m_daemon_login, proxy_address, upper_transaction_size_limit)) return false; // in case new wallet, this will force fast-refresh (pulling hashes instead of blocks) diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 43d85b704..caf1e9ed4 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -102,11 +102,12 @@ public: bool store(const std::string &path) override; std::string filename() const override; std::string keysFilename() const override; - bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false) override; + bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false, const std::string &proxy_address = "") override; bool connectToDaemon() override; ConnectionStatus connected() const override; void setTrustedDaemon(bool arg) override; bool trustedDaemon() const override; + bool setProxy(const std::string &address) override; uint64_t balance(uint32_t accountIndex = 0) const override; uint64_t unlockedBalance(uint32_t accountIndex = 0) const override; uint64_t blockChainHeight() const override; @@ -225,7 +226,7 @@ private: void stopRefresh(); bool isNewWallet() const; void pendingTxPostProcess(PendingTransactionImpl * pending); - bool doInit(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false); + bool doInit(const std::string &daemon_address, const std::string &proxy_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false); private: friend class PendingTransactionImpl; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index e2f96a069..50df7e5dd 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -533,9 +533,10 @@ struct Wallet * \param daemon_username * \param daemon_password * \param lightWallet - start wallet in light mode, connect to a openmonero compatible server. + * \param proxy_address - set proxy address, empty string to disable * \return - true on success */ - virtual bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false) = 0; + virtual bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false, const std::string &proxy_address = "") = 0; /*! * \brief createWatchOnly - Creates a watch only wallet @@ -594,6 +595,7 @@ struct Wallet virtual ConnectionStatus connected() const = 0; virtual void setTrustedDaemon(bool arg) = 0; virtual bool trustedDaemon() const = 0; + virtual bool setProxy(const std::string &address) = 0; virtual uint64_t balance(uint32_t accountIndex = 0) const = 0; uint64_t balanceAll() const { uint64_t result = 0; @@ -1298,6 +1300,9 @@ struct WalletManager std::string subdir, const char *buildtag = nullptr, const char *current_version = nullptr); + + //! sets proxy address, empty string to disable + virtual bool setProxy(const std::string &address) = 0; }; diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 69d17eb02..900fe91e5 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -375,6 +375,10 @@ std::tuple<bool, std::string, std::string, std::string, std::string> WalletManag return std::make_tuple(false, "", "", "", ""); } +bool WalletManagerImpl::setProxy(const std::string &address) +{ + return m_http_client.set_proxy(address); +} ///////////////////// WalletManagerFactory implementation ////////////////////// WalletManager *WalletManagerFactory::getWalletManager() diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index ca8e9ada2..2f603b0a9 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -30,7 +30,7 @@ #include "wallet/api/wallet2_api.h" -#include "net/http_client.h" +#include "net/http.h" #include <string> namespace Monero { @@ -92,11 +92,12 @@ public: bool startMining(const std::string &address, uint32_t threads = 1, bool background_mining = false, bool ignore_battery = true) override; bool stopMining() override; std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const override; + bool setProxy(const std::string &address) override; private: WalletManagerImpl() {} friend struct WalletManagerFactory; - epee::net_utils::http::http_simple_client m_http_client; + net::http::client m_http_client; std::string m_errorString; }; diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index fb07b42f0..303b576c7 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -27,7 +27,6 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "message_store.h" -#include <boost/archive/portable_binary_oarchive.hpp> #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/format.hpp> #include <boost/algorithm/string.hpp> @@ -182,8 +181,8 @@ bool message_store::signer_labels_complete() const void message_store::get_signer_config(std::string &signer_config) { std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << m_signers; + binary_archive<true> ar(oss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, m_signers), tools::error::wallet_internal_error, "Failed to serialize signer config"); signer_config = oss.str(); } @@ -194,8 +193,8 @@ void message_store::unpack_signer_config(const multisig_wallet_state &state, con { std::stringstream iss; iss << signer_config; - boost::archive::portable_binary_iarchive ar(iss); - ar >> signers; + binary_archive<false> ar(iss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, signers), tools::error::wallet_internal_error, "Failed to serialize signer config"); } catch (...) { @@ -364,8 +363,8 @@ size_t message_store::add_auto_config_data_message(const multisig_wallet_state & data.monero_address = me.monero_address; std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << data; + binary_archive<true> ar(oss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, data), tools::error::wallet_internal_error, "Failed to serialize auto config data"); return add_message(state, 0, message_type::auto_config_data, message_direction::out, oss.str()); } @@ -384,8 +383,8 @@ void message_store::process_auto_config_data_message(uint32_t id) { std::stringstream iss; iss << m.content; - boost::archive::portable_binary_iarchive ar(iss); - ar >> data; + binary_archive<false> ar(iss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, data), tools::error::wallet_internal_error, "Failed to serialize auto config data"); } catch (...) { @@ -745,8 +744,8 @@ std::string message_store::get_sanitized_text(const std::string &text, size_t ma void message_store::write_to_file(const multisig_wallet_state &state, const std::string &filename) { std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << *this; + binary_archive<true> ar(oss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, *this), tools::error::wallet_internal_error, "Failed to serialize MMS state"); std::string buf = oss.str(); crypto::chacha_key key; @@ -762,14 +761,14 @@ void message_store::write_to_file(const multisig_wallet_state &state, const std: write_file_data.encrypted_data = encrypted_data; std::stringstream file_oss; - boost::archive::portable_binary_oarchive file_ar(file_oss); - file_ar << write_file_data; + binary_archive<true> file_ar(file_oss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(file_ar, write_file_data), tools::error::wallet_internal_error, "Failed to serialize MMS state"); bool success = epee::file_io_utils::save_string_to_file(filename, file_oss.str()); THROW_WALLET_EXCEPTION_IF(!success, tools::error::file_save_error, filename); } -void message_store::read_from_file(const multisig_wallet_state &state, const std::string &filename) +void message_store::read_from_file(const multisig_wallet_state &state, const std::string &filename, bool load_deprecated_formats) { boost::system::error_code ignored_ec; bool file_exists = boost::filesystem::exists(filename, ignored_ec); @@ -785,17 +784,37 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std bool success = epee::file_io_utils::load_file_to_string(filename, buf); THROW_WALLET_EXCEPTION_IF(!success, tools::error::file_read_error, filename); + bool loaded = false; file_data read_file_data; try { std::stringstream iss; iss << buf; - boost::archive::portable_binary_iarchive ar(iss); - ar >> read_file_data; + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, read_file_data)) + if (::serialization::check_stream_state(ar)) + loaded = true; } - catch (const std::exception &e) + catch (...) {} + if (!loaded && load_deprecated_formats) + { + try + { + std::stringstream iss; + iss << buf; + boost::archive::portable_binary_iarchive ar(iss); + ar >> read_file_data; + loaded = true; + } + catch (const std::exception &e) + { + MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>: " << e.what()); + THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename); + } + } + if (!loaded) { - MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>: " << e.what()); + MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>"); THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename); } @@ -805,16 +824,36 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std decrypted_data.resize(read_file_data.encrypted_data.size()); crypto::chacha20(read_file_data.encrypted_data.data(), read_file_data.encrypted_data.size(), key, read_file_data.iv, &decrypted_data[0]); + loaded = false; try { std::stringstream iss; iss << decrypted_data; - boost::archive::portable_binary_iarchive ar(iss); - ar >> *this; + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, *this)) + if (::serialization::check_stream_state(ar)) + loaded = true; } - catch (const std::exception &e) + catch(...) {} + if (!loaded && load_deprecated_formats) + { + try + { + std::stringstream iss; + iss << decrypted_data; + boost::archive::portable_binary_iarchive ar(iss); + ar >> *this; + loaded = true; + } + catch (const std::exception &e) + { + MERROR("MMS file " << filename << " has bad structure: " << e.what()); + THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename); + } + } + if (!loaded) { - MERROR("MMS file " << filename << " has bad structure: " << e.what()); + MERROR("MMS file " << filename << " has bad structure"); THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename); } diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h index 9055fd776..0f53587d4 100644 --- a/src/wallet/message_store.h +++ b/src/wallet/message_store.h @@ -44,6 +44,9 @@ #include "common/command_line.h" #include "wipeable_string.h" #include "net/abstract_http_client.h" +#include "serialization/crypto.h" +#include "serialization/string.h" +#include "serialization/containers.h" #include "message_transporter.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -112,6 +115,24 @@ namespace mms uint32_t round; uint32_t signature_count; std::string transport_id; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + VARINT_FIELD(id) + VARINT_FIELD(type) + VARINT_FIELD(direction) + FIELD(content) + VARINT_FIELD(created) + VARINT_FIELD(modified) + VARINT_FIELD(sent) + VARINT_FIELD(signer_index) + FIELD(hash) + VARINT_FIELD(state) + VARINT_FIELD(wallet_height) + VARINT_FIELD(round) + VARINT_FIELD(signature_count) + FIELD(transport_id) + END_SERIALIZE() }; // "wallet_height" (for lack of a short name that would describe what it is about) // is the number of transfers present in the wallet at the time of message @@ -132,6 +153,21 @@ namespace mms std::string auto_config_transport_address; bool auto_config_running; + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(label) + FIELD(transport_address) + FIELD(monero_address_known) + FIELD(monero_address) + FIELD(me) + VARINT_FIELD(index) + FIELD(auto_config_token) + FIELD(auto_config_public_key) + FIELD(auto_config_secret_key) + FIELD(auto_config_transport_address) + FIELD(auto_config_running) + END_SERIALIZE() + authorized_signer() { monero_address_known = false; @@ -164,6 +200,13 @@ namespace mms std::string label; std::string transport_address; cryptonote::account_public_address monero_address; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(label) + FIELD(transport_address) + FIELD(monero_address) + END_SERIALIZE() }; // Overal .mms file structure, with the "message_store" object serialized to and @@ -174,6 +217,13 @@ namespace mms uint32_t file_version; crypto::chacha_iv iv; std::string encrypted_data; + + BEGIN_SERIALIZE_OBJECT() + FIELD(magic_string) + FIELD(file_version) + FIELD(iv) + FIELD(encrypted_data) + END_SERIALIZE() }; // The following struct provides info about the current state of a "wallet2" object @@ -198,6 +248,19 @@ namespace mms uint32_t multisig_rounds_passed; size_t num_transfer_details; std::string mms_file; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(address) + VARINT_FIELD(nettype) + FIELD(view_secret_key) + FIELD(multisig) + FIELD(multisig_is_ready) + FIELD(has_multisig_partial_key_images) + VARINT_FIELD(multisig_rounds_passed) + VARINT_FIELD(num_transfer_details) + FIELD(mms_file) + END_SERIALIZE() }; class message_store @@ -283,7 +346,7 @@ namespace mms void stop() { m_run.store(false, std::memory_order_relaxed); m_transporter.stop(); } void write_to_file(const multisig_wallet_state &state, const std::string &filename); - void read_from_file(const multisig_wallet_state &state, const std::string &filename); + void read_from_file(const multisig_wallet_state &state, const std::string &filename, bool load_deprecated_formats = false); template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) @@ -298,6 +361,18 @@ namespace mms a & m_auto_send; } + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(m_active) + VARINT_FIELD(m_num_authorized_signers) + VARINT_FIELD(m_nettype) + VARINT_FIELD(m_num_required_signers) + FIELD(m_signers) + FIELD(m_messages) + VARINT_FIELD(m_next_message_id) + FIELD(m_auto_send) + END_SERIALIZE() + static const char* message_type_to_string(message_type type); static const char* message_direction_to_string(message_direction direction); static const char* message_state_to_string(message_state state); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index d7ed3e999..5d78bb7b0 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -48,6 +48,7 @@ using namespace epee; #include "wallet_rpc_helpers.h" #include "wallet2.h" #include "cryptonote_basic/cryptonote_format_utils.h" +#include "net/parse.h" #include "rpc/core_rpc_server_commands_defs.h" #include "rpc/core_rpc_server_error_codes.h" #include "rpc/rpc_payment_signature.h" @@ -102,8 +103,8 @@ using namespace cryptonote; // used to target a given block weight (additional outputs may be added on top to build fee) #define TX_WEIGHT_TARGET(bytes) (bytes*2/3) -#define UNSIGNED_TX_PREFIX "Monero unsigned tx set\004" -#define SIGNED_TX_PREFIX "Monero signed tx set\004" +#define UNSIGNED_TX_PREFIX "Monero unsigned tx set\005" +#define SIGNED_TX_PREFIX "Monero signed tx set\005" #define MULTISIG_UNSIGNED_TX_PREFIX "Monero multisig unsigned tx set\001" #define RECENT_OUTPUT_RATIO (0.5) // 50% of outputs are from the recent zone @@ -440,30 +441,14 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl ); } - boost::asio::ip::tcp::endpoint proxy{}; + std::string proxy; if (use_proxy) { - namespace ip = boost::asio::ip; - - const auto proxy_address = command_line::get_arg(vm, opts.proxy); - - boost::string_ref proxy_port{proxy_address}; - boost::string_ref proxy_host = proxy_port.substr(0, proxy_port.rfind(":")); - if (proxy_port.size() == proxy_host.size()) - proxy_host = "127.0.0.1"; - else - proxy_port = proxy_port.substr(proxy_host.size() + 1); - - uint16_t port_value = 0; + proxy = command_line::get_arg(vm, opts.proxy); THROW_WALLET_EXCEPTION_IF( - !epee::string_tools::get_xtype_from_string(port_value, std::string{proxy_port}), + !net::get_tcp_endpoint(proxy), tools::error::wallet_internal_error, - std::string{"Invalid port specified for --"} + opts.proxy.name - ); - - boost::system::error_code error{}; - proxy = ip::tcp::endpoint{ip::address::from_string(std::string{proxy_host}, error), port_value}; - THROW_WALLET_EXCEPTION_IF(bool(error), tools::error::wallet_internal_error, std::string{"Invalid IP address specified for --"} + opts.proxy.name); + std::string{"Invalid address specified for --"} + opts.proxy.name); } boost::optional<bool> trusted_daemon; @@ -488,7 +473,10 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl } std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds, unattended)); - wallet->init(std::move(daemon_address), std::move(login), std::move(proxy), 0, *trusted_daemon, std::move(ssl_options)); + if (!wallet->init(std::move(daemon_address), std::move(login), std::move(proxy), 0, *trusted_daemon, std::move(ssl_options))) + { + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, tools::wallet2::tr("failed to initialize the wallet")); + } boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); wallet->get_message_store().set_options(vm); @@ -1125,7 +1113,7 @@ void wallet_device_callback::on_progress(const hw::device_progress& event) } wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory): - m_http_client(std::move(http_client_factory->create())), + m_http_client(http_client_factory->create()), m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), m_upper_transaction_weight_limit(0), @@ -1195,6 +1183,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std m_offline(false), m_rpc_version(0), m_export_format(ExportFormat::Binary), + m_load_deprecated_formats(false), m_credits_target(0) { set_rpc_client_secret_key(rct::rct2sk(rct::skGen())); @@ -1325,18 +1314,17 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u return ret; } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, boost::asio::ip::tcp::endpoint proxy, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_options_t ssl_options) +bool wallet2::set_proxy(const std::string &address) { + return m_http_client->set_proxy(address); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, const std::string &proxy_address, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_options_t ssl_options) +{ + CHECK_AND_ASSERT_MES(set_proxy(proxy_address), false, "failed to set proxy address"); m_checkpoints.init_default_checkpoints(m_nettype); m_is_initialized = true; m_upper_transaction_weight_limit = upper_transaction_weight_limit; - if (proxy != boost::asio::ip::tcp::endpoint{}) - { - epee::net_utils::http::abstract_http_client* abstract_http_client = m_http_client.get(); - epee::net_utils::http::http_simple_client* http_simple_client = dynamic_cast<epee::net_utils::http::http_simple_client*>(abstract_http_client); - CHECK_AND_ASSERT_MES(http_simple_client != nullptr, false, "http_simple_client must be used to set proxy"); - http_simple_client->set_connector(net::socks::connector{std::move(proxy)}); - } return set_daemon(daemon_address, daemon_login, trusted_daemon, std::move(ssl_options)); } //---------------------------------------------------------------------------------------------------- @@ -3916,6 +3904,9 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee: value2.SetInt(m_export_format); json.AddMember("export_format", value2, json.GetAllocator()); + value2.SetInt(m_load_deprecated_formats); + json.AddMember("load_deprecated_formats", value2, json.GetAllocator()); + value2.SetUint(1); json.AddMember("encrypted_secret_keys", value2, json.GetAllocator()); @@ -4085,6 +4076,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; m_original_keys_available = false; m_export_format = ExportFormat::Binary; + m_load_deprecated_formats = false; m_device_name = ""; m_device_derivation_path = ""; m_key_device_type = hw::device::device_type::SOFTWARE; @@ -4265,6 +4257,9 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, export_format, ExportFormat, Int, false, Binary); m_export_format = field_export_format; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, load_deprecated_formats, int, Int, false, false); + m_load_deprecated_formats = field_load_deprecated_formats; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_name, std::string, String, false, std::string()); if (m_device_name.empty()) { @@ -5058,7 +5053,7 @@ std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &passwor m_account.finalize_multisig(spend_public_key); m_multisig_signers = signers; - std::sort(m_multisig_signers.begin(), m_multisig_signers.end(), [](const crypto::public_key &e0, const crypto::public_key &e1){ return memcmp(&e0, &e1, sizeof(e0)); }); + std::sort(m_multisig_signers.begin(), m_multisig_signers.end(), [](const crypto::public_key &e0, const crypto::public_key &e1){ return memcmp(&e0, &e1, sizeof(e0)) < 0; }); ++m_multisig_rounds_passed; m_multisig_derivations.clear(); @@ -5614,10 +5609,26 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]); try { - std::stringstream iss; - iss << cache_data; - boost::archive::portable_binary_iarchive ar(iss); - ar >> *this; + bool loaded = false; + + try + { + std::stringstream iss; + iss << cache_data; + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, *this)) + if (::serialization::check_stream_state(ar)) + loaded = true; + } + catch(...) { } + + if (!loaded) + { + std::stringstream iss; + iss << cache_data; + boost::archive::portable_binary_iarchive ar(iss); + ar >> *this; + } } catch(...) { @@ -5714,7 +5725,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass try { if (use_fs) - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file, m_load_deprecated_formats); } catch (const std::exception &e) { @@ -5904,8 +5915,9 @@ boost::optional<wallet2::cache_file_data> wallet2::get_cache_file_data(const epe try { std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << *this; + binary_archive<true> ar(oss); + if (!::serialization::serialize(ar, *this)) + return boost::none; boost::optional<wallet2::cache_file_data> cache_file_data = (wallet2::cache_file_data) {}; cache_file_data.get().cache_data = oss.str(); @@ -6512,8 +6524,8 @@ void wallet2::commit_tx(pending_tx& ptx) add_unconfirmed_tx(ptx.tx, amount_in, dests, payment_id, ptx.change_dts.amount, ptx.construction_data.subaddr_account, ptx.construction_data.subaddr_indices); if (store_tx_info() && ptx.tx_key != crypto::null_skey) { - m_tx_keys.insert(std::make_pair(txid, ptx.tx_key)); - m_additional_tx_keys.insert(std::make_pair(txid, ptx.additional_tx_keys)); + m_tx_keys[txid] = ptx.tx_key; + m_additional_tx_keys[txid] = ptx.additional_tx_keys; } LOG_PRINT_L2("transaction " << txid << " generated ok and sent to daemon, key_images: [" << ptx.key_images << "]"); @@ -6567,10 +6579,11 @@ std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) c txs.transfers = export_outputs(); // save as binary std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); + binary_archive<true> ar(oss); try { - ar << txs; + if (!::serialization::serialize(ar, txs)) + return std::string(); } catch (...) { @@ -6614,6 +6627,11 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi s = s.substr(1); if (version == '\003') { + if (!m_load_deprecated_formats) + { + LOG_PRINT_L0("Not loading deprecated format"); + return false; + } try { std::istringstream iss(s); @@ -6628,6 +6646,11 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi } else if (version == '\004') { + if (!m_load_deprecated_formats) + { + LOG_PRINT_L0("Not loading deprecated format"); + return false; + } try { s = decrypt_with_view_secret_key(s); @@ -6649,6 +6672,26 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi return false; } } + else if (version == '\005') + { + try { s = decrypt_with_view_secret_key(s); } + catch(const std::exception &e) { LOG_PRINT_L0("Failed to decrypt unsigned tx: " << e.what()); return false; } + try + { + std::istringstream iss(s); + binary_archive<false> ar(iss); + if (!::serialization::serialize(ar, exported_txs)) + { + LOG_PRINT_L0("Failed to parse data from unsigned tx"); + return false; + } + } + catch (...) + { + LOG_PRINT_L0("Failed to parse data from unsigned tx"); + return false; + } + } else { LOG_PRINT_L0("Unsupported version in unsigned tx"); @@ -6702,8 +6745,8 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin if (store_tx_info() && tx_key != crypto::null_skey) { const crypto::hash txid = get_transaction_hash(ptx.tx); - m_tx_keys.insert(std::make_pair(txid, tx_key)); - m_additional_tx_keys.insert(std::make_pair(txid, additional_tx_keys)); + m_tx_keys[txid] = tx_key; + m_additional_tx_keys[txid] = additional_tx_keys; } std::string key_images; @@ -6846,10 +6889,11 @@ std::string wallet2::sign_tx_dump_to_str(unsigned_tx_set &exported_txs, std::vec // save as binary std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); + binary_archive<true> ar(oss); try { - ar << signed_txes; + if (!::serialization::serialize(ar, signed_txes)) + return std::string(); } catch(...) { @@ -6898,6 +6942,11 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too s = s.substr(1); if (version == '\003') { + if (!m_load_deprecated_formats) + { + LOG_PRINT_L0("Not loading deprecated format"); + return false; + } try { std::istringstream iss(s); @@ -6912,6 +6961,11 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too } else if (version == '\004') { + if (!m_load_deprecated_formats) + { + LOG_PRINT_L0("Not loading deprecated format"); + return false; + } try { s = decrypt_with_view_secret_key(s); @@ -6933,6 +6987,26 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too return false; } } + else if (version == '\005') + { + try { s = decrypt_with_view_secret_key(s); } + catch (const std::exception &e) { LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); return false; } + try + { + std::istringstream iss(s); + binary_archive<false> ar(iss); + if (!::serialization::serialize(ar, signed_txs)) + { + LOG_PRINT_L0("Failed to deserialize signed transaction"); + return false; + } + } + catch (const std::exception &e) + { + LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); + return false; + } + } else { LOG_PRINT_L0("Unsupported version in signed transaction"); @@ -6984,10 +7058,11 @@ std::string wallet2::save_multisig_tx(multisig_tx_set txs) // save as binary std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); + binary_archive<true> ar(oss); try { - ar << txs; + if (!::serialization::serialize(ar, txs)) + return std::string(); } catch (...) { @@ -7051,13 +7126,29 @@ bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx LOG_PRINT_L0("Failed to decrypt multisig tx data: " << e.what()); return false; } + bool loaded = false; try { std::istringstream iss(multisig_tx_st); - boost::archive::portable_binary_iarchive ar(iss); - ar >> exported_txs; + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, exported_txs)) + if (::serialization::check_stream_state(ar)) + loaded = true; } - catch (...) + catch (...) {} + try + { + if (!loaded && m_load_deprecated_formats) + { + std::istringstream iss(multisig_tx_st); + boost::archive::portable_binary_iarchive ar(iss); + ar >> exported_txs; + loaded = true; + } + } + catch(...) {} + + if (!loaded) { LOG_PRINT_L0("Failed to parse multisig tx data"); return false; @@ -7103,8 +7194,8 @@ bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported const crypto::hash txid = get_transaction_hash(ptx.tx); if (store_tx_info()) { - m_tx_keys.insert(std::make_pair(txid, ptx.tx_key)); - m_additional_tx_keys.insert(std::make_pair(txid, ptx.additional_tx_keys)); + m_tx_keys[txid] = ptx.tx_key; + m_additional_tx_keys[txid] = ptx.additional_tx_keys; } } } @@ -7224,8 +7315,8 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto const crypto::hash txid = get_transaction_hash(ptx.tx); if (store_tx_info()) { - m_tx_keys.insert(std::make_pair(txid, ptx.tx_key)); - m_additional_tx_keys.insert(std::make_pair(txid, ptx.additional_tx_keys)); + m_tx_keys[txid] = ptx.tx_key; + m_additional_tx_keys[txid] = ptx.additional_tx_keys; } txids.push_back(txid); } @@ -7460,7 +7551,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) getbh_req.client = get_client_signature(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, getbh_res, "getblockheadersrange", error::get_blocks_error, get_rpc_status(getbh_res.status)); - check_rpc_cost("/sendrawtransaction", getbh_res.credits, pre_call_credits, N * COST_PER_BLOCK_HEADER); + check_rpc_cost("/getblockheadersrange", getbh_res.credits, pre_call_credits, N * COST_PER_BLOCK_HEADER); } if (getbh_res.headers.size() != N) @@ -9544,8 +9635,8 @@ bool wallet2::light_wallet_parse_rct_str(const std::string& rct_string, const cr bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image, const crypto::public_key& tx_public_key, uint64_t out_index) { // Lookup key image from cache - std::map<uint64_t, crypto::key_image> index_keyimage_map; - std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> >::const_iterator found_pub_key = m_key_image_cache.find(tx_public_key); + serializable_map<uint64_t, crypto::key_image> index_keyimage_map; + serializable_unordered_map<crypto::public_key, serializable_map<uint64_t, crypto::key_image> >::const_iterator found_pub_key = m_key_image_cache.find(tx_public_key); if(found_pub_key != m_key_image_cache.end()) { // pub key found. key image for index cached? index_keyimage_map = found_pub_key->second; @@ -10915,7 +11006,7 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s return true; } //---------------------------------------------------------------------------------------------------- -void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys) +void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const boost::optional<cryptonote::account_public_address> &single_destination_subaddress) { // fetch tx from daemon and check if secret keys agree with corresponding public keys COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); @@ -10956,13 +11047,23 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ found = true; break; } + // when sent to a single subaddress, the derivation is different + if (single_destination_subaddress) + { + calculated_pub_key = rct::rct2pk(rct::scalarmultKey(rct::pk2rct(single_destination_subaddress->m_spend_public_key), rct::sk2rct(tx_key))); + if (calculated_pub_key == pub_key_field.pub_key) + { + found = true; + break; + } + } } THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Given tx secret key doesn't agree with the tx public key in the blockchain"); tx_extra_additional_pub_keys additional_tx_pub_keys; find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys); THROW_WALLET_EXCEPTION_IF(additional_tx_keys.size() != additional_tx_pub_keys.data.size(), error::wallet_internal_error, "The number of additional tx secret keys doesn't agree with the number of additional tx public keys in the blockchain" ); - m_tx_keys.insert(std::make_pair(txid, tx_key)); - m_additional_tx_keys.insert(std::make_pair(txid, additional_tx_keys)); + m_tx_keys[txid] = tx_key; + m_additional_tx_keys[txid] = additional_tx_keys; } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string &message) @@ -11425,7 +11526,7 @@ std::string wallet2::get_tx_proof(const cryptonote::transaction &tx, const crypt hwdev.generate_tx_proof(prefix_hash, tx_pub_key, address.m_view_public_key, boost::none, shared_secret[i], additional_tx_keys[i - 1], sig[i]); } } - sig_str = std::string("OutProofV1"); + sig_str = std::string("OutProofV2"); } else { @@ -11461,7 +11562,7 @@ std::string wallet2::get_tx_proof(const cryptonote::transaction &tx, const crypt hwdev.generate_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[i - 1], boost::none, shared_secret[i], a, sig[i]); } } - sig_str = std::string("InProofV1"); + sig_str = std::string("InProofV2"); } const size_t num_sigs = shared_secret.size(); @@ -11540,8 +11641,14 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const { + // InProofV1, InProofV2, OutProofV1, OutProofV2 const bool is_out = sig_str.substr(0, 3) == "Out"; - const std::string header = is_out ? "OutProofV1" : "InProofV1"; + const std::string header = is_out ? sig_str.substr(0,10) : sig_str.substr(0,9); + int version = 2; // InProofV2 + if (is_out && sig_str.substr(8,2) == "V1") version = 1; // OutProofV1 + else if (is_out) version = 2; // OutProofV2 + else if (sig_str.substr(7,2) == "V1") version = 1; // InProofV1 + const size_t header_len = header.size(); THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, "Signature header check error"); @@ -11588,27 +11695,27 @@ bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote if (is_out) { good_signature[0] = is_subaddress ? - crypto::check_tx_proof(prefix_hash, tx_pub_key, address.m_view_public_key, address.m_spend_public_key, shared_secret[0], sig[0]) : - crypto::check_tx_proof(prefix_hash, tx_pub_key, address.m_view_public_key, boost::none, shared_secret[0], sig[0]); + crypto::check_tx_proof(prefix_hash, tx_pub_key, address.m_view_public_key, address.m_spend_public_key, shared_secret[0], sig[0], version) : + crypto::check_tx_proof(prefix_hash, tx_pub_key, address.m_view_public_key, boost::none, shared_secret[0], sig[0], version); for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) { good_signature[i + 1] = is_subaddress ? - crypto::check_tx_proof(prefix_hash, additional_tx_pub_keys[i], address.m_view_public_key, address.m_spend_public_key, shared_secret[i + 1], sig[i + 1]) : - crypto::check_tx_proof(prefix_hash, additional_tx_pub_keys[i], address.m_view_public_key, boost::none, shared_secret[i + 1], sig[i + 1]); + crypto::check_tx_proof(prefix_hash, additional_tx_pub_keys[i], address.m_view_public_key, address.m_spend_public_key, shared_secret[i + 1], sig[i + 1], version) : + crypto::check_tx_proof(prefix_hash, additional_tx_pub_keys[i], address.m_view_public_key, boost::none, shared_secret[i + 1], sig[i + 1], version); } } else { good_signature[0] = is_subaddress ? - crypto::check_tx_proof(prefix_hash, address.m_view_public_key, tx_pub_key, address.m_spend_public_key, shared_secret[0], sig[0]) : - crypto::check_tx_proof(prefix_hash, address.m_view_public_key, tx_pub_key, boost::none, shared_secret[0], sig[0]); + crypto::check_tx_proof(prefix_hash, address.m_view_public_key, tx_pub_key, address.m_spend_public_key, shared_secret[0], sig[0], version) : + crypto::check_tx_proof(prefix_hash, address.m_view_public_key, tx_pub_key, boost::none, shared_secret[0], sig[0], version); for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) { good_signature[i + 1] = is_subaddress ? - crypto::check_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[i], address.m_spend_public_key, shared_secret[i + 1], sig[i + 1]) : - crypto::check_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[i], boost::none, shared_secret[i + 1], sig[i + 1]); + crypto::check_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[i], address.m_spend_public_key, shared_secret[i + 1], sig[i + 1], version) : + crypto::check_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[i], boost::none, shared_secret[i + 1], sig[i + 1], version); } } @@ -11727,7 +11834,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, } // collect all subaddress spend keys that received those outputs and generate their signatures - std::unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; + serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; for (const cryptonote::subaddress_index &index : subaddr_indices) { crypto::secret_key subaddr_spend_skey = m_account.get_keys().m_spend_secret_key; @@ -11744,9 +11851,10 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, // serialize & encode std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << proofs << subaddr_spendkeys; - return "ReserveProofV1" + tools::base58::encode(oss.str()); + binary_archive<true> ar(oss); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, proofs), error::wallet_internal_error, "Failed to serialize proof"); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, subaddr_spendkeys), error::wallet_internal_error, "Failed to serialize proof"); + return "ReserveProofV2" + tools::base58::encode(oss.str()); } bool wallet2::check_reserve_proof(const cryptonote::account_public_address &address, const std::string &message, const std::string &sig_str, uint64_t &total, uint64_t &spent) @@ -11755,19 +11863,39 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr THROW_WALLET_EXCEPTION_IF(!check_connection(&rpc_version), error::wallet_internal_error, "Failed to connect to daemon: " + get_daemon_address()); THROW_WALLET_EXCEPTION_IF(rpc_version < MAKE_CORE_RPC_VERSION(1, 0), error::wallet_internal_error, "Daemon RPC version is too old"); - static constexpr char header[] = "ReserveProofV1"; - THROW_WALLET_EXCEPTION_IF(!boost::string_ref{sig_str}.starts_with(header), error::wallet_internal_error, + static constexpr char header_v1[] = "ReserveProofV1"; + static constexpr char header_v2[] = "ReserveProofV2"; // assumes same length as header_v1 + THROW_WALLET_EXCEPTION_IF(!boost::string_ref{sig_str}.starts_with(header_v1) && !boost::string_ref{sig_str}.starts_with(header_v2), error::wallet_internal_error, "Signature header check error"); + int version = 2; // assume newest version + if (boost::string_ref{sig_str}.starts_with(header_v1)) + version = 1; + else if (boost::string_ref{sig_str}.starts_with(header_v2)) + version = 2; std::string sig_decoded; - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(std::strlen(header)), sig_decoded), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(std::strlen(header_v1)), sig_decoded), error::wallet_internal_error, "Signature decoding error"); - std::istringstream iss(sig_decoded); - boost::archive::portable_binary_iarchive ar(iss); + bool loaded = false; std::vector<reserve_proof_entry> proofs; - std::unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; - ar >> proofs >> subaddr_spendkeys; + serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; + try + { + std::istringstream iss(sig_decoded); + binary_archive<false> ar(iss); + if (::serialization::serialize_noeof(ar, proofs)) + if (::serialization::serialize_noeof(ar, subaddr_spendkeys)) + if (::serialization::check_stream_state(ar)) + loaded = true; + } + catch(...) {} + if (!loaded && m_load_deprecated_formats) + { + std::istringstream iss(sig_decoded); + boost::archive::portable_binary_iarchive ar(iss); + ar >> proofs >> subaddr_spendkeys.parent(); + } THROW_WALLET_EXCEPTION_IF(subaddr_spendkeys.count(address.m_spend_public_key) == 0, error::wallet_internal_error, "The given address isn't found in the proof"); @@ -11841,9 +11969,9 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); // check singature for shared secret - ok = crypto::check_tx_proof(prefix_hash, address.m_view_public_key, tx_pub_key, boost::none, proof.shared_secret, proof.shared_secret_sig); + ok = crypto::check_tx_proof(prefix_hash, address.m_view_public_key, tx_pub_key, boost::none, proof.shared_secret, proof.shared_secret_sig, version); if (!ok && additional_tx_pub_keys.size() == tx.vout.size()) - ok = crypto::check_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[proof.index_in_tx], boost::none, proof.shared_secret, proof.shared_secret_sig); + ok = crypto::check_tx_proof(prefix_hash, address.m_view_public_key, additional_tx_pub_keys[proof.index_in_tx], boost::none, proof.shared_secret, proof.shared_secret_sig, version); if (!ok) return false; @@ -12006,7 +12134,7 @@ std::string wallet2::get_description() const return ""; } -const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags() +const std::pair<serializable_map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags() { // ensure consistency if (m_account_tags.second.size() != get_num_subaddress_accounts()) @@ -12728,9 +12856,9 @@ std::string wallet2::export_outputs_to_str(bool all) const PERF_TIMER(export_outputs_to_str); std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - const auto& outputs = export_outputs(all); - ar << outputs; + binary_archive<true> ar(oss); + auto outputs = export_outputs(all); + THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, outputs), error::wallet_internal_error, "Failed to serialize output data"); std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; @@ -12842,23 +12970,39 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) } size_t imported_outputs = 0; + bool loaded = false; try { std::string body(data, headerlen); - std::stringstream iss; - iss << body; std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; try { - boost::archive::portable_binary_iarchive ar(iss); - ar >> outputs; + std::stringstream iss; + iss << body; + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, outputs)) + if (::serialization::check_stream_state(ar)) + loaded = true; } - catch (...) + catch (...) {} + + if (!loaded && m_load_deprecated_formats) { - iss.str(""); - iss << body; - boost::archive::binary_iarchive ar(iss); - ar >> outputs; + try + { + std::stringstream iss; + iss << body; + boost::archive::portable_binary_iarchive ar(iss); + ar >> outputs; + loaded = true; + } + catch (...) {} + } + + if (!loaded) + { + outputs.first = 0; + outputs.second = {}; } imported_outputs = import_outputs(outputs); @@ -13012,8 +13156,8 @@ cryptonote::blobdata wallet2::export_multisig() } std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << info; + binary_archive<true> ar(oss); + CHECK_AND_ASSERT_THROW_MES(::serialization::serialize(ar, info), "Failed to serialize multisig data"); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; @@ -13083,10 +13227,26 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) seen.insert(signer); std::string body(data, headerlen); - std::istringstream iss(body); std::vector<tools::wallet2::multisig_info> i; - boost::archive::portable_binary_iarchive ar(iss); - ar >> i; + + bool loaded = false; + try + { + std::istringstream iss(body); + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, i)) + if (::serialization::check_stream_state(ar)) + loaded = true; + } + catch(...) {} + if (!loaded && m_load_deprecated_formats) + { + std::istringstream iss(body); + boost::archive::portable_binary_iarchive ar(iss); + ar >> i; + loaded = true; + } + CHECK_AND_ASSERT_THROW_MES(loaded, "Failed to load output data"); MINFO(boost::format("%u outputs found") % boost::lexical_cast<std::string>(i.size())); info.push_back(std::move(i)); } @@ -13124,7 +13284,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) // sort by signer if (!info.empty() && !info.front().empty()) { - std::sort(info.begin(), info.end(), [](const std::vector<tools::wallet2::multisig_info> &i0, const std::vector<tools::wallet2::multisig_info> &i1){ return memcmp(&i0[0].m_signer, &i1[0].m_signer, sizeof(i0[0].m_signer)); }); + std::sort(info.begin(), info.end(), [](const std::vector<tools::wallet2::multisig_info> &i0, const std::vector<tools::wallet2::multisig_info> &i1){ return memcmp(&i0[0].m_signer, &i1[0].m_signer, sizeof(i0[0].m_signer)) < 0; }); } // first pass to determine where to detach the blockchain diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 712f91613..c526bac0a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -45,7 +45,7 @@ #include "cryptonote_basic/account.h" #include "cryptonote_basic/account_boost_serialization.h" #include "cryptonote_basic/cryptonote_basic_impl.h" -#include "net/http_client.h" +#include "net/http.h" #include "storages/http_abstract_invoke.h" #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_basic/cryptonote_format_utils.h" @@ -57,7 +57,10 @@ #include "ringct/rctTypes.h" #include "ringct/rctOps.h" #include "checkpoints/checkpoints.h" +#include "serialization/crypto.h" +#include "serialization/string.h" #include "serialization/pair.h" +#include "serialization/containers.h" #include "wallet_errors.h" #include "common/password.h" @@ -205,6 +208,13 @@ private: a & m_blockchain; } + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + VARINT_FIELD(m_offset) + FIELD(m_genesis) + FIELD(m_blockchain) + END_SERIALIZE() + private: size_t m_offset; crypto::hash m_genesis; @@ -269,7 +279,7 @@ private: static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1); - wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory = std::unique_ptr<epee::net_utils::http::http_simple_client_factory>(new epee::net_utils::http::http_simple_client_factory())); + wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory = std::unique_ptr<epee::net_utils::http::http_client_factory>(new net::http::client_factory())); ~wallet2(); struct multisig_info @@ -372,6 +382,19 @@ private: uint64_t m_timestamp; bool m_coinbase; cryptonote::subaddress_index m_subaddr_index; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(m_tx_hash) + VARINT_FIELD(m_amount) + FIELD(m_amounts) + VARINT_FIELD(m_fee) + VARINT_FIELD(m_block_height) + VARINT_FIELD(m_unlock_time) + VARINT_FIELD(m_timestamp) + FIELD(m_coinbase) + FIELD(m_subaddr_index) + END_SERIALIZE() }; struct address_tx : payment_details @@ -384,6 +407,12 @@ private: { payment_details m_pd; bool m_double_spend_seen; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(m_pd) + FIELD(m_double_spend_seen) + END_SERIALIZE() }; struct unconfirmed_transfer_details @@ -400,6 +429,21 @@ private: uint32_t m_subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> m_subaddr_indices; // set of address indices used as inputs in this transfer std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(m_tx) + VARINT_FIELD(m_amount_in) + VARINT_FIELD(m_amount_out) + VARINT_FIELD(m_change) + VARINT_FIELD(m_sent_time) + FIELD(m_dests) + FIELD(m_payment_id) + VARINT_FIELD(m_timestamp) + VARINT_FIELD(m_subaddr_account) + FIELD(m_subaddr_indices) + FIELD(m_rings) + END_SERIALIZE() }; struct confirmed_transfer_details @@ -419,6 +463,21 @@ private: confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(crypto::null_hash), m_timestamp(0), m_unlock_time(0), m_subaddr_account((uint32_t)-1) {} confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height): m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices), m_rings(utd.m_rings) {} + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + VARINT_FIELD(m_amount_in) + VARINT_FIELD(m_amount_out) + VARINT_FIELD(m_change) + VARINT_FIELD(m_block_height) + FIELD(m_dests) + FIELD(m_payment_id) + VARINT_FIELD(m_timestamp) + VARINT_FIELD(m_unlock_time) + VARINT_FIELD(m_subaddr_account) + FIELD(m_subaddr_indices) + FIELD(m_rings) + END_SERIALIZE() }; struct tx_construction_data @@ -451,7 +510,7 @@ private: }; typedef std::vector<transfer_details> transfer_container; - typedef std::unordered_multimap<crypto::hash, payment_details> payment_container; + typedef serializable_unordered_multimap<crypto::hash, payment_details> payment_container; struct multisig_sig { @@ -460,6 +519,15 @@ private: std::unordered_set<rct::key> used_L; std::unordered_set<crypto::public_key> signing_keys; rct::multisig_out msout; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(sigs) + FIELD(ignore) + FIELD(used_L) + FIELD(signing_keys) + FIELD(msout) + END_SERIALIZE() }; // The convention for destinations is: @@ -502,13 +570,26 @@ private: { std::vector<tx_construction_data> txes; std::pair<size_t, wallet2::transfer_container> transfers; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(txes) + FIELD(transfers) + END_SERIALIZE() }; struct signed_tx_set { std::vector<pending_tx> ptx; std::vector<crypto::key_image> key_images; - std::unordered_map<crypto::public_key, crypto::key_image> tx_key_images; + serializable_unordered_map<crypto::public_key, crypto::key_image> tx_key_images; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(ptx) + FIELD(key_images) + FIELD(tx_key_images) + END_SERIALIZE() }; struct multisig_tx_set @@ -543,7 +624,7 @@ private: FIELD(cache_data) END_SERIALIZE() }; - + // GUI Address book struct address_book_row { @@ -552,6 +633,15 @@ private: std::string m_description; bool m_is_subaddress; bool m_has_payment_id; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(m_address) + FIELD(m_payment_id) + FIELD(m_description) + FIELD(m_is_subaddress) + FIELD(m_has_payment_id) + END_SERIALIZE() }; struct reserve_proof_entry @@ -562,6 +652,16 @@ private: crypto::key_image key_image; crypto::signature shared_secret_sig; crypto::signature key_image_sig; + + BEGIN_SERIALIZE_OBJECT() + VERSION_FIELD(0) + FIELD(txid) + VARINT_FIELD(index_in_tx) + FIELD(shared_secret) + FIELD(key_image) + FIELD(shared_secret_sig) + FIELD(key_image_sig) + END_SERIALIZE() }; typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry; @@ -753,13 +853,14 @@ private: bool deinit(); bool init(std::string daemon_address = "http://localhost:8080", boost::optional<epee::net_utils::http::login> daemon_login = boost::none, - boost::asio::ip::tcp::endpoint proxy = {}, + const std::string &proxy = "", uint64_t upper_transaction_weight_limit = 0, bool trusted_daemon = true, epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); bool set_daemon(std::string daemon_address = "http://localhost:8080", boost::optional<epee::net_utils::http::login> daemon_login = boost::none, bool trusted_daemon = true, epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); + bool set_proxy(const std::string &address); void stop() { m_run.store(false, std::memory_order_relaxed); m_message_store.stop(); } @@ -917,6 +1018,7 @@ private: { std::vector<crypto::hash> blockchain; a & blockchain; + m_blockchain.clear(); for (const auto &b: blockchain) { m_blockchain.push_back(b); @@ -928,25 +1030,25 @@ private: } a & m_transfers; a & m_account_public_address; - a & m_key_images; + a & m_key_images.parent(); if(ver < 6) return; - a & m_unconfirmed_txs; + a & m_unconfirmed_txs.parent(); if(ver < 7) return; - a & m_payments; + a & m_payments.parent(); if(ver < 8) return; - a & m_tx_keys; + a & m_tx_keys.parent(); if(ver < 9) return; - a & m_confirmed_txs; + a & m_confirmed_txs.parent(); if(ver < 11) return; a & dummy_refresh_height; if(ver < 12) return; - a & m_tx_notes; + a & m_tx_notes.parent(); if(ver < 13) return; if (ver < 17) @@ -954,6 +1056,7 @@ private: // we're loading an old version, where m_unconfirmed_payments was a std::map std::unordered_map<crypto::hash, payment_details> m; a & m; + m_unconfirmed_payments.clear(); for (std::unordered_map<crypto::hash, payment_details>::const_iterator i = m.begin(); i != m.end(); ++i) m_unconfirmed_payments.insert(std::make_pair(i->first, pool_payment_details{i->second, false})); } @@ -962,6 +1065,7 @@ private: if(ver < 15) { // we're loading an older wallet without a pubkey map, rebuild it + m_pub_keys.clear(); for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details &td = m_transfers[i]; @@ -971,7 +1075,7 @@ private: } return; } - a & m_pub_keys; + a & m_pub_keys.parent(); if(ver < 16) return; a & m_address_book; @@ -982,6 +1086,7 @@ private: // we're loading an old version, where m_unconfirmed_payments payload was payment_details std::unordered_multimap<crypto::hash, payment_details> m; a & m; + m_unconfirmed_payments.clear(); for (const auto &i: m) m_unconfirmed_payments.insert(std::make_pair(i.first, pool_payment_details{i.second, false})); } @@ -991,20 +1096,20 @@ private: a & m_scanned_pool_txs[1]; if (ver < 20) return; - a & m_subaddresses; + a & m_subaddresses.parent(); std::unordered_map<cryptonote::subaddress_index, crypto::public_key> dummy_subaddresses_inv; a & dummy_subaddresses_inv; a & m_subaddress_labels; - a & m_additional_tx_keys; + a & m_additional_tx_keys.parent(); if(ver < 21) return; - a & m_attributes; + a & m_attributes.parent(); if(ver < 22) return; - a & m_unconfirmed_payments; + a & m_unconfirmed_payments.parent(); if(ver < 23) return; - a & m_account_tags; + a & (std::pair<std::map<std::string, std::string>, std::vector<std::string>>&)m_account_tags; if(ver < 24) return; a & m_ring_history_saved; @@ -1013,18 +1118,48 @@ private: a & m_last_block_reward; if(ver < 26) return; - a & m_tx_device; + a & m_tx_device.parent(); if(ver < 27) return; a & m_device_last_key_image_sync; if(ver < 28) return; - a & m_cold_key_images; + a & m_cold_key_images.parent(); if(ver < 29) return; a & m_rpc_client_secret_key; } + BEGIN_SERIALIZE_OBJECT() + MAGIC_FIELD("monero wallet cache") + VERSION_FIELD(0) + FIELD(m_blockchain) + FIELD(m_transfers) + FIELD(m_account_public_address) + FIELD(m_key_images) + FIELD(m_unconfirmed_txs) + FIELD(m_payments) + FIELD(m_tx_keys) + FIELD(m_confirmed_txs) + FIELD(m_tx_notes) + FIELD(m_unconfirmed_payments) + FIELD(m_pub_keys) + FIELD(m_address_book) + FIELD(m_scanned_pool_txs[0]) + FIELD(m_scanned_pool_txs[1]) + FIELD(m_subaddresses) + FIELD(m_subaddress_labels) + FIELD(m_additional_tx_keys) + FIELD(m_attributes) + FIELD(m_account_tags) + FIELD(m_ring_history_saved) + FIELD(m_last_block_reward) + FIELD(m_tx_device) + FIELD(m_device_last_key_image_sync) + FIELD(m_cold_key_images) + FIELD(m_rpc_client_secret_key) + END_SERIALIZE() + /*! * \brief Check if wallet keys and bin files exist * \param file_path Wallet file path @@ -1096,6 +1231,8 @@ private: void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } const ExportFormat & export_format() const { return m_export_format; } inline void set_export_format(const ExportFormat& export_format) { m_export_format = export_format; } + bool load_deprecated_formats() const { return m_load_deprecated_formats; } + void load_deprecated_formats(bool load) { m_load_deprecated_formats = load; } bool persistent_rpc_client_id() const { return m_persistent_rpc_client_id; } void persistent_rpc_client_id(bool persistent) { m_persistent_rpc_client_id = persistent; } void auto_mine_for_rpc_payment_threshold(float threshold) { m_auto_mine_for_rpc_payment_threshold = threshold; } @@ -1106,7 +1243,7 @@ private: void credits_target(uint64_t threshold) { m_credits_target = threshold; } bool get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; - void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); + void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const boost::optional<cryptonote::account_public_address> &single_destination_subaddress = boost::none); bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys); void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); @@ -1187,7 +1324,7 @@ private: * \brief Get the list of registered account tags. * \return first.Key=(tag's name), first.Value=(tag's label), second[i]=(i-th account's tag) */ - const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& get_account_tags(); + const std::pair<serializable_map<std::string, std::string>, std::vector<std::string>>& get_account_tags(); /*! * \brief Set a tag to the given accounts. * \param account_indices Indices of accounts. @@ -1527,28 +1664,28 @@ private: std::string m_mms_file; const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client; hashchain m_blockchain; - std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; - std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; - std::unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; - std::unordered_map<crypto::hash, crypto::secret_key> m_tx_keys; + serializable_unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; + serializable_unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; + serializable_unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; + serializable_unordered_map<crypto::hash, crypto::secret_key> m_tx_keys; cryptonote::checkpoints m_checkpoints; - std::unordered_map<crypto::hash, std::vector<crypto::secret_key>> m_additional_tx_keys; + serializable_unordered_map<crypto::hash, std::vector<crypto::secret_key>> m_additional_tx_keys; transfer_container m_transfers; payment_container m_payments; - std::unordered_map<crypto::key_image, size_t> m_key_images; - std::unordered_map<crypto::public_key, size_t> m_pub_keys; + serializable_unordered_map<crypto::key_image, size_t> m_key_images; + serializable_unordered_map<crypto::public_key, size_t> m_pub_keys; cryptonote::account_public_address m_account_public_address; - std::unordered_map<crypto::public_key, cryptonote::subaddress_index> m_subaddresses; + serializable_unordered_map<crypto::public_key, cryptonote::subaddress_index> m_subaddresses; std::vector<std::vector<std::string>> m_subaddress_labels; - std::unordered_map<crypto::hash, std::string> m_tx_notes; - std::unordered_map<std::string, std::string> m_attributes; + serializable_unordered_map<crypto::hash, std::string> m_tx_notes; + serializable_unordered_map<std::string, std::string> m_attributes; std::vector<tools::wallet2::address_book_row> m_address_book; - std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags; + std::pair<serializable_map<std::string, std::string>, std::vector<std::string>> m_account_tags; uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; - std::unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; + serializable_unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; std::atomic<bool> m_run; @@ -1615,7 +1752,7 @@ private: uint64_t m_credits_target; // Aux transaction data from device - std::unordered_map<crypto::hash, std::string> m_tx_device; + serializable_unordered_map<crypto::hash, std::string> m_tx_device; // Light wallet bool m_light_wallet; /* sends view key to daemon for scanning */ @@ -1629,7 +1766,7 @@ private: // We save the info from the first call in m_light_wallet_address_txs for easier lookup. std::unordered_map<crypto::hash, address_tx> m_light_wallet_address_txs; // store calculated key image for faster lookup - std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> > m_key_image_cache; + serializable_unordered_map<crypto::public_key, serializable_map<uint64_t, crypto::key_image> > m_key_image_cache; std::string m_ring_database; bool m_ring_history_saved; @@ -1656,6 +1793,7 @@ private: std::unique_ptr<wallet_device_callback> m_device_callback; ExportFormat m_export_format; + bool m_load_deprecated_formats; static boost::mutex default_daemon_address_lock; static std::string default_daemon_address; @@ -2052,7 +2190,7 @@ namespace boost a & x.key_images; if (ver < 1) return; - a & x.tx_key_images; + a & x.tx_key_images.parent(); } template <class Archive> diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index fcaaf7616..0ed749cb7 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -80,7 +80,7 @@ namespace return pwd_container; } //------------------------------------------------------------------------------------------------------------------------------ - void set_confirmations(tools::wallet_rpc::transfer_entry &entry, uint64_t blockchain_height, uint64_t block_reward) + void set_confirmations(tools::wallet_rpc::transfer_entry &entry, uint64_t blockchain_height, uint64_t block_reward, uint64_t unlock_time) { if (entry.height >= blockchain_height || (entry.height == 0 && (!strcmp(entry.type.c_str(), "pending") || !strcmp(entry.type.c_str(), "pool")))) entry.confirmations = 0; @@ -91,6 +91,18 @@ namespace entry.suggested_confirmations_threshold = 0; else entry.suggested_confirmations_threshold = (entry.amount + block_reward - 1) / block_reward; + + if (unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) + { + if (unlock_time > blockchain_height) + entry.suggested_confirmations_threshold = std::max(entry.suggested_confirmations_threshold, unlock_time - blockchain_height); + } + else + { + const uint64_t now = time(NULL); + if (unlock_time > now) + entry.suggested_confirmations_threshold = std::max(entry.suggested_confirmations_threshold, (unlock_time - now + DIFFICULTY_TARGET_V2 - 1) / DIFFICULTY_TARGET_V2); + } } } @@ -335,7 +347,7 @@ namespace tools entry.subaddr_index = pd.m_subaddr_index; entry.subaddr_indices.push_back(pd.m_subaddr_index); entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); - set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward(), pd.m_unlock_time); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const tools::wallet2::confirmed_transfer_details &pd) @@ -365,7 +377,7 @@ namespace tools for (uint32_t i: pd.m_subaddr_indices) entry.subaddr_indices.push_back({pd.m_subaddr_account, i}); entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); - set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward(), pd.m_unlock_time); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const tools::wallet2::unconfirmed_transfer_details &pd) @@ -396,7 +408,7 @@ namespace tools for (uint32_t i: pd.m_subaddr_indices) entry.subaddr_indices.push_back({pd.m_subaddr_account, i}); entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); - set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward(), pd.m_tx.unlock_time); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &payment_id, const tools::wallet2::pool_payment_details &ppd) @@ -419,7 +431,7 @@ namespace tools entry.subaddr_index = pd.m_subaddr_index; entry.subaddr_indices.push_back(pd.m_subaddr_index); entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); - set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward(), pd.m_unlock_time); } //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, const connection_context *ctx) @@ -826,10 +838,11 @@ namespace tools static std::string ptx_to_string(const tools::wallet2::pending_tx &ptx) { std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); + binary_archive<true> ar(oss); try { - ar << ptx; + if (!::serialization::serialize(ar, const_cast<tools::wallet2::pending_tx&>(ptx))) + return ""; } catch (...) { @@ -1538,14 +1551,31 @@ namespace tools return false; } + bool loaded = false; tools::wallet2::pending_tx ptx; + try { std::istringstream iss(blob); - boost::archive::portable_binary_iarchive ar(iss); - ar >> ptx; + binary_archive<false> ar(iss); + if (::serialization::serialize(ar, ptx)) + loaded = true; } - catch (...) + catch(...) {} + + if (!loaded && !m_restricted) + { + try + { + std::istringstream iss(blob); + boost::archive::portable_binary_iarchive ar(iss); + ar >> ptx; + loaded = true; + } + catch (...) {} + } + + if (!loaded) { er.code = WALLET_RPC_ERROR_CODE_BAD_TX_METADATA; er.message = "Failed to parse tx metadata."; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index ae861d177..5b2536bf1 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 18 +#define WALLET_RPC_VERSION_MINOR 19 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ed5a3b9e3..c601b93ed 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,6 +28,8 @@ # # Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +set(MONERO_WALLET_CRYPTO_BENCH "auto" CACHE STRING "Select wallet crypto libraries for benchmarking") + # The docs say this only affects grouping in IDEs set(folder "tests") set(TEST_DATA_DIR "${CMAKE_CURRENT_LIST_DIR}/data") @@ -118,6 +120,41 @@ add_test( NAME hash-target COMMAND hash-target-tests) +# +# Configure wallet crypto benchmark +# +if (${MONERO_WALLET_CRYPTO_BENCH} STREQUAL "auto") + set(MONERO_WALLET_CRYPTO_BENCH "cn") + monero_crypto_autodetect(AVAILABLE BEST) + if (DEFINED AVAILABLE) + list(APPEND MONERO_WALLET_CRYPTO_BENCH ${AVAILABLE}) + endif () + message("Wallet crypto bench is using ${MONERO_WALLET_CRYPTO_BENCH}") +endif () + +list(REMOVE_DUPLICATES MONERO_WALLET_CRYPTO_BENCH) +list(REMOVE_ITEM MONERO_WALLET_CRYPTO_BENCH "cn") # always used for comparison +set(MONERO_WALLET_CRYPTO_BENCH_NAMES "(cn)") +foreach(BENCH IN LISTS MONERO_WALLET_CRYPTO_BENCH) + monero_crypto_valid(${BENCH} VALID) + if (NOT VALID) + message(FATAL_ERROR "Invalid MONERO_WALLET_CRYPTO_BENCH option ${BENCH}") + endif () + + monero_crypto_get_target(${BENCH} BENCH_LIBRARY) + list(APPEND BENCH_OBJECTS $<TARGET_OBJECTS:${BENCH_LIBRARY}>) + + monero_crypto_get_namespace(${BENCH} BENCH_NAMESPACE) + set(MONERO_WALLET_CRYPTO_BENCH_NAMES "${MONERO_WALLET_CRYPTO_BENCH_NAMES}(${BENCH_NAMESPACE})") +endforeach () + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/benchmark.h.in" "${MONERO_GENERATED_HEADERS_DIR}/tests/benchmark.h") +add_executable(monero-wallet-crypto-bench benchmark.cpp ${BENCH_OBJECTS}) +target_link_libraries(monero-wallet-crypto-bench cncrypto) + +add_test(NAME wallet-crypto-bench COMMAND monero-wallet-crypto-bench) + + set(enabled_tests core_tests difficulty diff --git a/tests/benchmark.cpp b/tests/benchmark.cpp new file mode 100644 index 000000000..0461f4c11 --- /dev/null +++ b/tests/benchmark.cpp @@ -0,0 +1,437 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "tests/benchmark.h" + +#include <boost/fusion/adapted/std_tuple.hpp> +#include <boost/fusion/algorithm/iteration/fold.hpp> +#include <boost/preprocessor/seq/enum.hpp> +#include <boost/preprocessor/seq/for_each.hpp> +#include <boost/preprocessor/seq/seq.hpp> +#include <boost/preprocessor/seq.hpp> +#include <boost/preprocessor/stringize.hpp> +#include <boost/spirit/include/karma_char.hpp> +#include <boost/spirit/include/karma_format.hpp> +#include <boost/spirit/include/karma_repeat.hpp> +#include <boost/spirit/include/karma_right_alignment.hpp> +#include <boost/spirit/include/karma_sequence.hpp> +#include <boost/spirit/include/karma_string.hpp> +#include <boost/spirit/include/karma_uint.hpp> +#include <boost/spirit/include/qi_char.hpp> +#include <boost/spirit/include/qi_list.hpp> +#include <boost/spirit/include/qi_parse.hpp> +#include <boost/spirit/include/qi_uint.hpp> +#include <chrono> +#include <cstring> +#include <functional> +#include <iostream> +#include <stdexcept> +#include <string> +#include <tuple> +#include <type_traits> +#include <utility> +#include <vector> + +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_basic.h" +#include "monero/crypto/amd64-64-24k.h" +#include "monero/crypto/amd64-51-30k.h" + +#define CHECK(...) \ + if(!( __VA_ARGS__ )) \ + throw std::runtime_error{ \ + "TEST FAILED (line " \ + BOOST_PP_STRINGIZE( __LINE__ ) \ + "): " \ + BOOST_PP_STRINGIZE( __VA_ARGS__ ) \ + } + +//! Define function that forwards arguments to `crypto::func`. +#define FORWARD_FUNCTION(func) \ + template<typename... T> \ + static bool func (T&&... args) \ + { \ + return ::crypto:: func (std::forward<T>(args)...); \ + } + +#define CRYPTO_FUNCTION(library, func) \ + BOOST_PP_CAT(BOOST_PP_CAT(monero_crypto_, library), func) + +#define CRYPTO_BENCHMARK(r, _, library) \ + struct library \ + { \ + static constexpr const char* name() noexcept { return BOOST_PP_STRINGIZE(library); } \ + static bool generate_key_derivation(const ::crypto::public_key &tx_pub, const ::crypto::secret_key &view_sec, ::crypto::key_derivation &out) \ + { \ + return CRYPTO_FUNCTION(library, _generate_key_derivation) (out.data, tx_pub.data, view_sec.data) == 0; \ + } \ + static bool derive_subaddress_public_key(const ::crypto::public_key &spend_pub, const ::crypto::key_derivation &d, std::size_t index, ::crypto::public_key &out) \ + { \ + ::crypto::ec_scalar scalar; \ + ::crypto::derivation_to_scalar(d, index, scalar); \ + return CRYPTO_FUNCTION(library, _generate_subaddress_public_key) (out.data, spend_pub.data, scalar.data) == 0; \ + } \ + }; + + +namespace +{ + //! Default number of iterations for benchmark timing. + constexpr const unsigned default_iterations = 1000; + + //! \return Byte compare two objects of `T`. + template<typename T> + bool compare(const T& lhs, const T& rhs) noexcept + { + static_assert(!epee::has_padding<T>(), "type might have padding"); + return std::memcmp(std::addressof(lhs), std::addressof(rhs), sizeof(T)) == 0; + } + + //! Benchmark default monero crypto library - a re-arranged ref10 implementation. + struct cn + { + static constexpr const char* name() noexcept { return "cn"; } + FORWARD_FUNCTION( generate_key_derivation ); + FORWARD_FUNCTION( derive_subaddress_public_key ); + }; + + // Define functions for every library except for `cn` which is the head library. + BOOST_PP_SEQ_FOR_EACH(CRYPTO_BENCHMARK, _, BOOST_PP_SEQ_TAIL(BENCHMARK_LIBRARIES)); + + // All enabled benchmark libraries + using enabled_libraries = std::tuple<BOOST_PP_SEQ_ENUM(BENCHMARK_LIBRARIES)>; + + + //! Callable that runs a benchmark against all enabled libraries + template<typename R> + struct run_benchmark + { + using result = R; + + template<typename B> + result operator()(result out, const B benchmark) const + { + using inner_result = typename B::result; + out.push_back({boost::fusion::fold(enabled_libraries{}, inner_result{}, benchmark), benchmark.name()}); + std::sort(out.back().first.begin(), out.back().first.end()); + return out; + } + }; + + //! Run 0+ benchmarks against all enabled libraries + template<typename R, typename... B> + R run_benchmarks(B&&... benchmarks) + { + auto out = boost::fusion::fold(std::make_tuple(std::forward<B>(benchmarks)...), R{}, run_benchmark<R>{}); + std::sort(out.begin(), out.end()); + return out; + } + + //! Run a suite of benchmarks - allows for comparison against a subset of benchmarks + template<typename S> + std::pair<typename S::result, std::string> run_suite(const S& suite) + { + return {suite(), suite.name()}; + } + + //! Arguments given to every crypto library being benchmarked. + struct bench_args + { + explicit bench_args(unsigned iterations) + : iterations(iterations), one(), two() + { + crypto::generate_keys(one.pub, one.sec, one.sec, false); + crypto::generate_keys(two.pub, two.sec, two.sec, false); + } + + const unsigned iterations; + cryptonote::keypair one; + cryptonote::keypair two; + }; + + /*! Tests the ECDH step used for monero txes where the tx-pub is always + de-compressed into a table every time. */ + struct tx_pub_standard + { + using result = std::vector<std::pair<std::chrono::steady_clock::duration, std::string>>; + static constexpr const char* name() noexcept { return "standard"; } + + const bench_args args; + + template<typename L> + result operator()(result out, const L library) const + { + crypto::key_derivation us; + crypto::key_derivation them; + CHECK(crypto::generate_key_derivation(args.one.pub, args.two.sec, them)); + CHECK(library.generate_key_derivation(args.one.pub, args.two.sec, us)); + CHECK(compare(us, them)); + + unsigned i = 0; + for (unsigned j = 0; j < 100; ++j) + i += library.generate_key_derivation(args.one.pub, args.two.sec, us); + CHECK(i == 100); + + i = 0; + const auto start = std::chrono::steady_clock::now(); + for (unsigned j = 0; j < args.iterations; ++j) + i += library.generate_key_derivation(args.one.pub, args.two.sec, us); + const auto end = std::chrono::steady_clock::now(); + CHECK(i == args.iterations); + CHECK(compare(us, them)); + + out.push_back({end - start, library.name()}); + return out; + } + }; + + //! Tests various possible optimizations for tx ECDH-step. + struct tx_pub_suite + { + using result = std::vector<std::pair<tx_pub_standard::result, std::string>>; + static constexpr const char* name() noexcept { return "generate_key_derivation step"; } + + const bench_args args; + + result operator()() const + { + return run_benchmarks<result>(tx_pub_standard{args}); + } + }; + + /*! Tests the shared-secret to output-key step used for monero txes where + the users spend-public is always de-compressed. */ + struct output_pub_standard + { + using result = std::vector<std::pair<std::chrono::steady_clock::duration, std::string>>; + static constexpr const char* name() noexcept { return "standard"; } + + const bench_args args; + + template<typename L> + result operator()(result out, const L library) const + { + crypto::key_derivation derived; + crypto::public_key us; + crypto::public_key them; + CHECK(crypto::generate_key_derivation(args.one.pub, args.two.sec, derived)); + CHECK(library.derive_subaddress_public_key(args.two.pub, derived, 0, us)); + CHECK(crypto::derive_subaddress_public_key(args.two.pub, derived, 0, them)); + CHECK(compare(us, them)); + + unsigned i = 0; + for (unsigned j = 0; j < 100; ++j) + i += library.derive_subaddress_public_key(args.two.pub, derived, j, us); + CHECK(i == 100); + + i = 0; + const auto start = std::chrono::steady_clock::now(); + for (unsigned j = 0; j < args.iterations; ++j) + i += library.derive_subaddress_public_key(args.two.pub, derived, j, us); + const auto end = std::chrono::steady_clock::now(); + CHECK(i == args.iterations); + + out.push_back({end - start, library.name()}); + return out; + } + }; + + //! Tests various possible optimizations for shared-secret to output-key step. + struct output_pub_suite + { + using result = std::vector<std::pair<output_pub_standard::result, std::string>>; + static constexpr const char* name() noexcept { return "derive_subaddress_public_key step"; } + + const bench_args args; + + result operator()() const + { + return run_benchmarks<result>(output_pub_standard{args}); + } + }; + + struct tx_bench_args + { + const bench_args main; + unsigned outputs; + }; + + /*! Simulates "standard" tx scanning where a tx-pubkey is de-compressed into + a table and user spend-public is de-compressed, every time. */ + struct tx_standard + { + using result = std::vector<std::pair<std::chrono::steady_clock::duration, std::string>>; + static constexpr const char* name() noexcept { return "standard"; } + + const tx_bench_args args; + + template<typename L> + result operator()(result out, const L library) const + { + crypto::key_derivation derived_us; + crypto::key_derivation derived_them; + crypto::public_key us; + crypto::public_key them; + CHECK(library.generate_key_derivation(args.main.one.pub, args.main.two.sec, derived_us)); + CHECK(crypto::generate_key_derivation(args.main.one.pub, args.main.two.sec, derived_them)); + CHECK(library.derive_subaddress_public_key(args.main.two.pub, derived_us, 0, us)); + CHECK(crypto::derive_subaddress_public_key(args.main.two.pub, derived_them, 0, them)); + CHECK(compare(us, them)); + + unsigned i = 0; + for (unsigned j = 0; j < 100; ++j) + { + i += library.generate_key_derivation(args.main.one.pub, args.main.two.sec, derived_us); + i += library.derive_subaddress_public_key(args.main.two.pub, derived_us, j, us); + } + CHECK(i == 200); + + i = 0; + const auto start = std::chrono::steady_clock::now(); + for (unsigned j = 0; j < args.main.iterations; ++j) + { + i += library.generate_key_derivation(args.main.one.pub, args.main.two.sec, derived_us); + for (unsigned k = 0; k < args.outputs; ++k) + i += library.derive_subaddress_public_key(args.main.two.pub, derived_us, k, us); + } + const auto end = std::chrono::steady_clock::now(); + CHECK(i == args.main.iterations + args.main.iterations * args.outputs); + + out.push_back({end - start, library.name()}); + return out; + } + }; + + //! Tests various possible optimizations for tx scanning. + struct tx_suite + { + using result = std::vector<std::pair<output_pub_standard::result, std::string>>; + std::string name() const { return "Transactions with " + std::to_string(args.outputs) + " outputs"; } + + const tx_bench_args args; + + result operator()() const + { + return run_benchmarks<result>(tx_standard{args}); + + } + }; + + std::chrono::steady_clock::duration print(const tx_pub_standard::result& leaf, std::ostream& out, unsigned depth) + { + namespace karma = boost::spirit::karma; + const std::size_t align = leaf.empty() ? + 0 : std::to_string(leaf.back().first.count()).size(); + const auto best = leaf.empty() ? + std::chrono::steady_clock::duration::max() : leaf.front().first; + for (auto const& entry : leaf) + { + out << karma::format(karma::repeat(depth ? depth - 1 : 0)["| "]) << '|'; + out << karma::format((karma::right_align(std::min(20u - depth, 20u), '-')["> " << karma::string]), entry.second); + out << " => " << karma::format((karma::right_align(align)[karma::uint_]), entry.first.count()); + out << " ns (+"; + out << (double((entry.first - best).count()) / best.count()) * 100 << "%)" << std::endl; + } + out << karma::format(karma::repeat(depth ? depth - 1 : 0)["| "]) << std::endl; + return best; + } + + template<typename T> + std::chrono::steady_clock::duration + print(const std::vector<std::pair<T, std::string>>& node, std::ostream& out, unsigned depth) + { + auto best = std::chrono::steady_clock::duration::max(); + for (auto const& entry : node) + { + std::stringstream buffer{}; + auto last = print(entry.first, buffer, depth + 1); + if (last != std::chrono::steady_clock::duration::max()) + { + namespace karma = boost::spirit::karma; + best = std::min(best, last); + out << karma::format(karma::repeat(depth)["|-"]); + out << "+ " << entry.second << ' '; + out << last.count() << " ns (+"; + out << (double((last - best).count()) / best.count()) * 100 << "%)" << std::endl; + out << buffer.str(); + } + } + return best; + } +} // anonymous namespace + +int main(int argc, char** argv) +{ + using results = std::vector<std::pair<tx_pub_suite::result, std::string>>; + try + { + unsigned iterations = default_iterations; + std::vector<unsigned> nums{}; + if (2 <= argc) iterations = std::stoul(argv[1]); + if (3 <= argc) + { + namespace qi = boost::spirit::qi; + if (!qi::parse(argv[2], argv[2] + strlen(argv[2]), (qi::uint_ % ','), nums)) + throw std::runtime_error{"bad tx outputs string"}; + } + else + { + nums = {2, 4}; + } + std::sort(nums.begin(), nums.end()); + nums.erase(std::unique(nums.begin(), nums.end()), nums.end()); + + std::cout << "Running benchmark using " << iterations << " iterations" << std::endl; + + const bench_args args{iterations}; + + results val{}; + + std::cout << "Transaction Component Benchmarks" << std::endl; + std::cout << "--------------------------------" << std::endl; + val.push_back(run_suite(tx_pub_suite{args})); + val.push_back(run_suite(output_pub_suite{args})); + std::sort(val.begin(), val.end()); + print(val, std::cout, 0); + + val.clear(); + std::cout << "Transaction Benchmarks" << std::endl; + std::cout << "----------------------" << std::endl; + for (const unsigned num : nums) + val.push_back(run_suite(tx_suite{{args, num}})); + std::sort(val.begin(), val.end()); + print(val, std::cout, 0); + } + catch (const std::exception& e) + { + std::cerr << "Error: " << e.what() << std::endl; + return 1; + } + return 0; +} + diff --git a/tests/benchmark.h.in b/tests/benchmark.h.in new file mode 100644 index 000000000..b13ea30b7 --- /dev/null +++ b/tests/benchmark.h.in @@ -0,0 +1,5 @@ +#pragma once + +// A Boost PP sequence +#define BENCHMARK_LIBRARIES @MONERO_WALLET_CRYPTO_BENCH_NAMES@ + diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index edaa9b20a..8b6135510 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -35,8 +35,6 @@ #include <iostream> #include <stdint.h> -#include <boost/archive/binary_oarchive.hpp> -#include <boost/archive/binary_iarchive.hpp> #include <boost/program_options.hpp> #include <boost/optional.hpp> #include <boost/serialization/vector.hpp> diff --git a/tests/data/fuzz/cold-outputs/OUTPUTS1 b/tests/data/fuzz/cold-outputs/OUTPUTS1 Binary files differdeleted file mode 100644 index f449f61ad..000000000 --- a/tests/data/fuzz/cold-outputs/OUTPUTS1 +++ /dev/null diff --git a/tests/data/fuzz/cold-outputs/OUTPUTS2 b/tests/data/fuzz/cold-outputs/OUTPUTS2 Binary files differdeleted file mode 100644 index 33cf39024..000000000 --- a/tests/data/fuzz/cold-outputs/OUTPUTS2 +++ /dev/null diff --git a/tests/data/fuzz/cold-outputs/out-all-6 b/tests/data/fuzz/cold-outputs/out-all-6 Binary files differnew file mode 100644 index 000000000..d24fc604f --- /dev/null +++ b/tests/data/fuzz/cold-outputs/out-all-6 diff --git a/tests/data/fuzz/cold-outputs/out-none-6 b/tests/data/fuzz/cold-outputs/out-none-6 Binary files differnew file mode 100644 index 000000000..c5390590c --- /dev/null +++ b/tests/data/fuzz/cold-outputs/out-none-6 diff --git a/tests/data/fuzz/cold-transaction/CTX1 b/tests/data/fuzz/cold-transaction/CTX1 Binary files differindex 0afecedbc..4b9ee45dc 100644 --- a/tests/data/fuzz/cold-transaction/CTX1 +++ b/tests/data/fuzz/cold-transaction/CTX1 diff --git a/tests/functional_tests/check_missing_rpc_methods.py b/tests/functional_tests/check_missing_rpc_methods.py index 6fadebf9b..0eedd6d0f 100644 --- a/tests/functional_tests/check_missing_rpc_methods.py +++ b/tests/functional_tests/check_missing_rpc_methods.py @@ -46,5 +46,6 @@ for module in modules: name = name[1:] if not hasattr(module['object'], name): print('Error: %s API method %s does not have a matching function' % (module['name'], name)) + error = True sys.exit(1 if error else 0) diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 5f2a3d077..3be62c0ca 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -10,7 +10,7 @@ import string import os USAGE = 'usage: functional_tests_rpc.py <python> <srcdir> <builddir> [<tests-to-run> | all]' -DEFAULT_TESTS = ['address_book', 'bans', 'blockchain', 'cold_signing', 'daemon_info', 'get_output_distribution', 'integrated_address', 'mining', 'multisig', 'proofs', 'rpc_payment', 'sign_message', 'transfer', 'txpool', 'uri', 'validate_address', 'wallet'] +DEFAULT_TESTS = ['address_book', 'bans', 'blockchain', 'cold_signing', 'daemon_info', 'get_output_distribution', 'integrated_address', 'mining', 'multisig', 'p2p', 'proofs', 'rpc_payment', 'sign_message', 'transfer', 'txpool', 'uri', 'validate_address', 'wallet'] try: python = sys.argv[1] srcdir = sys.argv[2] @@ -34,18 +34,32 @@ try: except: tests = DEFAULT_TESTS -N_MONERODS = 2 -N_WALLETS = 4 +# a main offline monerod, does most of the tests +# a restricted RPC monerod setup with RPC payment +# two local online monerods connected to each other +N_MONERODS = 4 + +# 4 wallets connected to the main offline monerod +# a wallet connected to the first local online monerod +N_WALLETS = 5 + WALLET_DIRECTORY = builddir + "/functional-tests-directory" DIFFICULTY = 10 -monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", str(DIFFICULTY), "--offline", "--no-igd", "--p2p-bind-port", "monerod_p2p_port", "--rpc-bind-port", "monerod_rpc_port", "--zmq-rpc-bind-port", "monerod_zmq_port", "--non-interactive", "--disable-dns-checkpoints", "--check-updates", "disabled", "--rpc-ssl", "disabled", "--log-level", "1"] +monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", str(DIFFICULTY), "--no-igd", "--p2p-bind-port", "monerod_p2p_port", "--rpc-bind-port", "monerod_rpc_port", "--zmq-rpc-bind-port", "monerod_zmq_port", "--non-interactive", "--disable-dns-checkpoints", "--check-updates", "disabled", "--rpc-ssl", "disabled", "--data-dir", "monerod_data_dir", "--log-level", "1"] monerod_extra = [ - [], - ["--rpc-payment-address", "44SKxxLQw929wRF6BA9paQ1EWFshNnKhXM3qz6Mo3JGDE2YG3xyzVutMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUiqhcwR", "--rpc-payment-difficulty", str(DIFFICULTY), "--rpc-payment-credits", "5000", "--data-dir", builddir + "/functional-tests-directory/monerod1"], + ["--offline"], + ["--rpc-payment-address", "44SKxxLQw929wRF6BA9paQ1EWFshNnKhXM3qz6Mo3JGDE2YG3xyzVutMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUiqhcwR", "--rpc-payment-difficulty", str(DIFFICULTY), "--rpc-payment-credits", "5000", "--offline"], + ["--add-exclusive-node", "127.0.0.1:18283"], + ["--add-exclusive-node", "127.0.0.1:18282"], ] -wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--daemon-port", "18180", "--log-level", "1"] +wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--log-level", "1"] wallet_extra = [ + ["--daemon-port", "18180"], + ["--daemon-port", "18180"], + ["--daemon-port", "18180"], + ["--daemon-port", "18180"], + ["--daemon-port", "18182"], ] command_lines = [] @@ -54,7 +68,7 @@ outputs = [] ports = [] for i in range(N_MONERODS): - command_lines.append([str(18180+i) if x == "monerod_rpc_port" else str(18280+i) if x == "monerod_p2p_port" else str(18380+i) if x == "monerod_zmq_port" else x for x in monerod_base]) + command_lines.append([str(18180+i) if x == "monerod_rpc_port" else str(18280+i) if x == "monerod_p2p_port" else str(18380+i) if x == "monerod_zmq_port" else builddir + "/functional-tests-directory/monerod" + str(i) if x == "monerod_data_dir" else x for x in monerod_base]) if i < len(monerod_extra): command_lines[-1] += monerod_extra[i] outputs.append(open(builddir + '/tests/functional_tests/monerod' + str(i) + '.log', 'a+')) @@ -109,6 +123,9 @@ if not all_open: kill() sys.exit(1) +# online daemons need some time to connect to peers to be ready +time.sleep(2) + PASS = [] FAIL = [] for test in tests: diff --git a/tests/functional_tests/p2p.py b/tests/functional_tests/p2p.py new file mode 100755 index 000000000..f36e9c0b1 --- /dev/null +++ b/tests/functional_tests/p2p.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2018 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function +import time + +"""Test daemon P2P +""" + +from framework.daemon import Daemon +from framework.wallet import Wallet + +class P2PTest(): + def run_test(self): + self.reset() + self.create() + self.mine(80) + self.test_p2p_reorg() + self.test_p2p_tx_propagation() + + def reset(self): + print('Resetting blockchain') + daemon = Daemon() + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) + daemon.flush_txpool() + + def create(self): + print('Creating wallet') + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + self.wallet = Wallet(idx = 4) + # close the wallet if any, will throw if none is loaded + try: self.wallet.close_wallet() + except: pass + res = self.wallet.restore_deterministic_wallet(seed = seed) + + def mine(self, blocks): + assert blocks >= 1 + + print("Generating", blocks, 'blocks') + + daemon = Daemon(idx = 2) + + # generate blocks + res_generateblocks = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', blocks) + + def test_p2p_reorg(self): + print('Testing P2P reorg') + daemon2 = Daemon(idx = 2) + daemon3 = Daemon(idx = 3) + + # give sync some time + time.sleep(1) + + res = daemon2.get_info() + height = res.height + assert height > 0 + top_block_hash = res.top_block_hash + assert len(top_block_hash) == 64 + + res = daemon3.get_info() + assert res.height == height + assert res.top_block_hash == top_block_hash + + # disconnect daemons and mine separately on both + daemon2.out_peers(0) + daemon3.out_peers(0) + + res = daemon2.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 2) + res = daemon3.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 3) + + res = daemon2.get_info() + assert res.height == height + 2 + daemon2_top_block_hash = res.top_block_hash + assert daemon2_top_block_hash != top_block_hash + res = daemon3.get_info() + assert res.height == height + 3 + daemon3_top_block_hash = res.top_block_hash + assert daemon3_top_block_hash != top_block_hash + assert daemon3_top_block_hash != daemon2_top_block_hash + + # reconnect, daemon2 will now switch to daemon3's chain + daemon2.out_peers(8) + daemon3.out_peers(8) + time.sleep(10) + res = daemon2.get_info() + assert res.height == height + 3 + assert res.top_block_hash == daemon3_top_block_hash + + # disconect, mine on daemon2 again more than daemon3 + daemon2.out_peers(0) + daemon3.out_peers(0) + + res = daemon2.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 3) + res = daemon3.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 2) + + res = daemon2.get_info() + assert res.height == height + 6 + daemon2_top_block_hash = res.top_block_hash + assert daemon2_top_block_hash != top_block_hash + res = daemon3.get_info() + assert res.height == height + 5 + daemon3_top_block_hash = res.top_block_hash + assert daemon3_top_block_hash != top_block_hash + assert daemon3_top_block_hash != daemon2_top_block_hash + + # reconnect, daemon3 will now switch to daemon2's chain + daemon2.out_peers(8) + daemon3.out_peers(8) + time.sleep(5) + res = daemon3.get_info() + assert res.height == height + 6 + assert res.top_block_hash == daemon2_top_block_hash + + def test_p2p_tx_propagation(self): + print('Testing P2P tx propagation') + daemon2 = Daemon(idx = 2) + daemon3 = Daemon(idx = 3) + + for daemon in [daemon2, daemon3]: + res = daemon.get_transaction_pool_hashes() + assert not 'tx_hashes' in res or len(res.tx_hashes) == 0 + + self.wallet.refresh() + res = self.wallet.get_balance() + + dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000} + res = self.wallet.transfer([dst]) + assert len(res.tx_hash) == 32*2 + txid = res.tx_hash + + time.sleep(5) + + for daemon in [daemon2, daemon3]: + res = daemon.get_transaction_pool_hashes() + assert len(res.tx_hashes) == 1 + assert res.tx_hashes[0] == txid + + +if __name__ == '__main__': + P2PTest().run_test() diff --git a/tests/functional_tests/proofs.py b/tests/functional_tests/proofs.py index 5f23f7ea4..e58d29f94 100755 --- a/tests/functional_tests/proofs.py +++ b/tests/functional_tests/proofs.py @@ -130,13 +130,13 @@ class ProofsTest(): sending_address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' receiving_address = '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW' res = self.wallet[0].get_tx_proof(txid, sending_address, 'foo'); - assert res.signature.startswith('InProof'); + assert res.signature.startswith('InProofV2'); signature0i = res.signature res = self.wallet[0].get_tx_proof(txid, receiving_address, 'bar'); - assert res.signature.startswith('OutProof'); + assert res.signature.startswith('OutProofV2'); signature0o = res.signature res = self.wallet[1].get_tx_proof(txid, receiving_address, 'baz'); - assert res.signature.startswith('InProof'); + assert res.signature.startswith('InProofV2'); signature1 = res.signature res = self.wallet[0].check_tx_proof(txid, sending_address, 'foo', signature0i); @@ -219,6 +219,23 @@ class ProofsTest(): except: ok = True assert ok or not res.good + + # Test bad cross-version verification + ok = False + try: res = self.wallet[0].check_tx_proof(txid, sending_address, 'foo', signature0i.replace('ProofV2','ProofV1')); + except: ok = True + assert ok or not res.good + + ok = False + try: res = self.wallet[0].check_tx_proof(txid, receiving_address, 'bar', signature0o.replace('ProofV2','ProofV1')); + except: ok = True + assert ok or not res.good + + ok = False + try: res = self.wallet[1].check_tx_proof(txid, receiving_address, 'baz', signature1.replace('ProofV2','ProofV1')); + except: ok = True + assert ok or not res.good + def check_spend_proof(self, txid): daemon = Daemon() @@ -270,7 +287,7 @@ class ProofsTest(): balance1 = res.balance res = self.wallet[0].get_reserve_proof(all_ = True, message = 'foo') - assert res.signature.startswith('ReserveProof') + assert res.signature.startswith('ReserveProofV2') signature = res.signature for i in range(2): res = self.wallet[i].check_reserve_proof(address = address0, message = 'foo', signature = signature) @@ -287,9 +304,15 @@ class ProofsTest(): except: ok = True assert ok or not res.good + # Test bad cross-version verification + ok = False + try: res = self.wallet[i].check_reserve_proof(address = address0, message = 'foo', signature = signature.replace('ProofV2','ProofV1')) + except: ok = True + assert ok or not res.good + amount = int(balance0 / 10) res = self.wallet[0].get_reserve_proof(all_ = False, amount = amount, message = 'foo') - assert res.signature.startswith('ReserveProof') + assert res.signature.startswith('ReserveProofV2') signature = res.signature for i in range(2): res = self.wallet[i].check_reserve_proof(address = address0, message = 'foo', signature = signature) @@ -306,6 +329,12 @@ class ProofsTest(): except: ok = True assert ok or not res.good + # Test bad cross-version verification + ok = False + try: res = self.wallet[i].check_reserve_proof(address = address0, message = 'foo', signature = signature.replace('ProofV2','ProofV1')) + except: ok = True + assert ok or not res.good + ok = False try: self.wallet[0].get_reserve_proof(all_ = False, amount = balance0 + 1, message = 'foo') except: ok = True diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index fea235079..2698a36ba 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -40,22 +40,22 @@ BEGIN_INIT_SIMPLE_FUZZER() static tools::wallet2 local_wallet; wallet = &local_wallet; - static const char * const spendkey_hex = "0b4f47697ec99c3de6579304e5f25c68b07afbe55b71d99620bf6cbf4e45a80f"; + static const char * const spendkey_hex = "f285d4ac9e66271256fc7cde0d3d6b36f66efff6ccd766706c408e86f4997a0d"; crypto::secret_key spendkey; epee::string_tools::hex_to_pod(spendkey_hex, spendkey); - wallet->init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + wallet->init("", boost::none, "", 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet->set_subaddress_lookahead(1, 1); wallet->generate("", "", spendkey, true, false); END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::string s = std::string("\x01\x16serialization::archive") + std::string((const char*)buf, len); + std::string s((const char*)buf, len); std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; std::stringstream iss; iss << s; - boost::archive::portable_binary_iarchive ar(iss); - ar >> outputs; + binary_archive<false> ar(iss); + ::serialization::serialize(ar, outputs); size_t n_outputs = wallet->import_outputs(outputs); std::cout << boost::lexical_cast<std::string>(n_outputs) << " outputs imported" << std::endl; END_SIMPLE_FUZZER() diff --git a/tests/fuzz/cold-transaction.cpp b/tests/fuzz/cold-transaction.cpp index 32c84ac74..135343704 100644 --- a/tests/fuzz/cold-transaction.cpp +++ b/tests/fuzz/cold-transaction.cpp @@ -40,22 +40,22 @@ BEGIN_INIT_SIMPLE_FUZZER() static tools::wallet2 local_wallet; wallet = &local_wallet; - static const char * const spendkey_hex = "0b4f47697ec99c3de6579304e5f25c68b07afbe55b71d99620bf6cbf4e45a80f"; + static const char * const spendkey_hex = "f285d4ac9e66271256fc7cde0d3d6b36f66efff6ccd766706c408e86f4997a0d"; crypto::secret_key spendkey; epee::string_tools::hex_to_pod(spendkey_hex, spendkey); - wallet->init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + wallet->init("", boost::none, "", 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet->set_subaddress_lookahead(1, 1); wallet->generate("", "", spendkey, true, false); END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::string s = std::string("\x01\x16serialization::archive") + std::string((const char*)buf, len); + std::string s((const char*)buf, len); tools::wallet2::unsigned_tx_set exported_txs; std::stringstream iss; iss << s; - boost::archive::portable_binary_iarchive ar(iss); - ar >> exported_txs; + binary_archive<false> ar(iss); + ::serialization::serialize(ar, exported_txs); std::vector<tools::wallet2::pending_tx> ptx; bool success = wallet->sign_tx(exported_txs, "/tmp/cold-transaction-test-signed", ptx); std::cout << (success ? "signed" : "error") << std::endl; diff --git a/tests/fuzz/signature.cpp b/tests/fuzz/signature.cpp index 8f528b20e..2a3e65c25 100644 --- a/tests/fuzz/signature.cpp +++ b/tests/fuzz/signature.cpp @@ -45,7 +45,7 @@ BEGIN_INIT_SIMPLE_FUZZER() crypto::secret_key spendkey; epee::string_tools::hex_to_pod(spendkey_hex, spendkey); - wallet->init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + wallet->init("", boost::none, "", 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet->set_subaddress_lookahead(1, 1); wallet->generate("", "", spendkey, true, false); diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index ef0477888..a5984b2c9 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -83,6 +83,7 @@ set(unit_tests_sources test_peerlist.cpp test_protocol_pack.cpp threadpool.cpp + tx_proof.cpp hardfork.cpp unbound.cpp uri.cpp @@ -109,6 +110,7 @@ target_link_libraries(unit_tests cryptonote_protocol cryptonote_core daemon_messages + daemon_rpc_server blockchain_db lmdb_lib rpc diff --git a/tests/unit_tests/json_serialization.cpp b/tests/unit_tests/json_serialization.cpp index 5873d0ab6..f76199e57 100644 --- a/tests/unit_tests/json_serialization.cpp +++ b/tests/unit_tests/json_serialization.cpp @@ -15,7 +15,7 @@ #include "serialization/json_object.h" -namespace +namespace test { cryptonote::transaction make_miner_transaction(cryptonote::account_public_address const& to) @@ -82,7 +82,10 @@ namespace return tx; } +} +namespace +{ template<typename T> T test_json(const T& value) { @@ -94,7 +97,7 @@ namespace rapidjson::Document doc; doc.Parse(reinterpret_cast<const char*>(buffer.data()), buffer.size()); - if (doc.HasParseError() || !doc.IsObject()) + if (doc.HasParseError()) { throw cryptonote::json::PARSE_FAIL(); } @@ -105,11 +108,26 @@ namespace } } // anonymous +TEST(JsonSerialization, VectorBytes) +{ + EXPECT_EQ(std::vector<std::uint8_t>{}, test_json(std::vector<std::uint8_t>{})); + EXPECT_EQ(std::vector<std::uint8_t>{0x00}, test_json(std::vector<std::uint8_t>{0x00})); +} + +TEST(JsonSerialization, InvalidVectorBytes) +{ + rapidjson::Document doc; + doc.SetString("1"); + + std::vector<std::uint8_t> out; + EXPECT_THROW(cryptonote::json::fromJsonValue(doc, out), cryptonote::json::BAD_INPUT); +} + TEST(JsonSerialization, MinerTransaction) { cryptonote::account_base acct; acct.generate(); - const auto miner_tx = make_miner_transaction(acct.get_keys().m_account_address); + const auto miner_tx = test::make_miner_transaction(acct.get_keys().m_account_address); crypto::hash tx_hash{}; ASSERT_TRUE(cryptonote::get_transaction_hash(miner_tx, tx_hash)); @@ -137,8 +155,8 @@ TEST(JsonSerialization, RegularTransaction) cryptonote::account_base acct2; acct2.generate(); - const auto miner_tx = make_miner_transaction(acct1.get_keys().m_account_address); - const auto tx = make_transaction( + const auto miner_tx = test::make_miner_transaction(acct1.get_keys().m_account_address); + const auto tx = test::make_transaction( acct1.get_keys(), {miner_tx}, {acct2.get_keys().m_account_address}, false, false ); @@ -168,8 +186,8 @@ TEST(JsonSerialization, RingctTransaction) cryptonote::account_base acct2; acct2.generate(); - const auto miner_tx = make_miner_transaction(acct1.get_keys().m_account_address); - const auto tx = make_transaction( + const auto miner_tx = test::make_miner_transaction(acct1.get_keys().m_account_address); + const auto tx = test::make_transaction( acct1.get_keys(), {miner_tx}, {acct2.get_keys().m_account_address}, true, false ); @@ -199,8 +217,8 @@ TEST(JsonSerialization, BulletproofTransaction) cryptonote::account_base acct2; acct2.generate(); - const auto miner_tx = make_miner_transaction(acct1.get_keys().m_account_address); - const auto tx = make_transaction( + const auto miner_tx = test::make_miner_transaction(acct1.get_keys().m_account_address); + const auto tx = test::make_transaction( acct1.get_keys(), {miner_tx}, {acct2.get_keys().m_account_address}, true, true ); diff --git a/tests/unit_tests/json_serialization.h b/tests/unit_tests/json_serialization.h new file mode 100644 index 000000000..2d8267261 --- /dev/null +++ b/tests/unit_tests/json_serialization.h @@ -0,0 +1,42 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace test +{ + cryptonote::transaction make_miner_transaction(cryptonote::account_public_address const& to); + + cryptonote::transaction + make_transaction( + cryptonote::account_keys const& from, + std::vector<cryptonote::transaction> const& sources, + std::vector<cryptonote::account_public_address> const& destinations, + bool rct, + bool bulletproof); +} diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index d9d273837..15563e764 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -680,6 +680,76 @@ TEST_F(levin_notify, local_without_padding) } } +TEST_F(levin_notify, forward_without_padding) +{ + cryptonote::levin::notify notifier = make_notifier(0, true, false); + + for (unsigned count = 0; count < 10; ++count) + add_connection(count % 2 == 0); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'f'); + txs[1].resize(200, 'e'); + + std::vector<cryptonote::blobdata> sorted_txs = txs; + std::sort(sorted_txs.begin(), sorted_txs.end()); + + ASSERT_EQ(10u, contexts_.size()); + bool has_stemmed = false; + bool has_fluffed = false; + while (!has_stemmed || !has_fluffed) + { + auto context = contexts_.begin(); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), events_, cryptonote::relay_method::forward)); + + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + const bool is_stem = events_.has_stem_txes(); + EXPECT_EQ(txs, events_.take_relayed(is_stem ? cryptonote::relay_method::stem : cryptonote::relay_method::fluff)); + + if (!is_stem) + { + notifier.run_fluff(); + ASSERT_LT(0u, io_service_.poll()); + } + + std::size_t send_count = 0; + EXPECT_EQ(0u, context->process_send_queue()); + for (++context; context != contexts_.end(); ++context) + { + const std::size_t sent = context->process_send_queue(); + if (sent && is_stem) + EXPECT_EQ(1u, (context - contexts_.begin()) % 2); + send_count += sent; + } + + EXPECT_EQ(is_stem ? 1u : 9u, send_count); + ASSERT_EQ(is_stem ? 1u : 9u, receiver_.notified_size()); + for (unsigned count = 0; count < (is_stem ? 1u : 9u); ++count) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + if (is_stem) + EXPECT_EQ(txs, notification.txs); + else + EXPECT_EQ(sorted_txs, notification.txs); + EXPECT_TRUE(notification._.empty()); + EXPECT_EQ(!is_stem, notification.dandelionpp_fluff); + } + + has_stemmed |= is_stem; + has_fluffed |= !is_stem; + notifier.run_epoch(); + } +} + TEST_F(levin_notify, block_without_padding) { cryptonote::levin::notify notifier = make_notifier(0, true, false); @@ -918,6 +988,73 @@ TEST_F(levin_notify, local_with_padding) } } +TEST_F(levin_notify, forward_with_padding) +{ + cryptonote::levin::notify notifier = make_notifier(0, true, true); + + for (unsigned count = 0; count < 10; ++count) + add_connection(count % 2 == 0); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + txs[1].resize(200, 'f'); + + ASSERT_EQ(10u, contexts_.size()); + bool has_stemmed = false; + bool has_fluffed = false; + while (!has_stemmed || !has_fluffed) + { + auto context = contexts_.begin(); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), events_, cryptonote::relay_method::forward)); + + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + const bool is_stem = events_.has_stem_txes(); + EXPECT_EQ(txs, events_.take_relayed(is_stem ? cryptonote::relay_method::stem : cryptonote::relay_method::fluff)); + + if (!is_stem) + { + notifier.run_fluff(); + ASSERT_LT(0u, io_service_.poll()); + } + + std::size_t send_count = 0; + EXPECT_EQ(0u, context->process_send_queue()); + for (++context; context != contexts_.end(); ++context) + { + const std::size_t sent = context->process_send_queue(); + if (sent && is_stem) + { + EXPECT_EQ(1u, (context - contexts_.begin()) % 2); + EXPECT_FALSE(context->is_incoming()); + } + send_count += sent; + } + + EXPECT_EQ(is_stem ? 1u : 9u, send_count); + ASSERT_EQ(is_stem ? 1u : 9u, receiver_.notified_size()); + for (unsigned count = 0; count < (is_stem ? 1u : 9u); ++count) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_FALSE(notification._.empty()); + EXPECT_EQ(!is_stem, notification.dandelionpp_fluff); + } + + has_stemmed |= is_stem; + has_fluffed |= !is_stem; + notifier.run_epoch(); + } +} + TEST_F(levin_notify, block_with_padding) { cryptonote::levin::notify notifier = make_notifier(0, true, true); @@ -1021,7 +1158,7 @@ TEST_F(levin_notify, private_fluff_without_padding) auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; EXPECT_EQ(txs, notification.txs); EXPECT_TRUE(notification._.empty()); - EXPECT_FALSE(notification.dandelionpp_fluff); + EXPECT_TRUE(notification.dandelionpp_fluff); } } } @@ -1057,7 +1194,7 @@ TEST_F(levin_notify, private_stem_without_padding) io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); - EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff)); + EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem)); EXPECT_EQ(0u, context->process_send_queue()); for (++context; context != contexts_.end(); ++context) @@ -1072,7 +1209,7 @@ TEST_F(levin_notify, private_stem_without_padding) auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; EXPECT_EQ(txs, notification.txs); EXPECT_TRUE(notification._.empty()); - EXPECT_FALSE(notification.dandelionpp_fluff); + EXPECT_TRUE(notification.dandelionpp_fluff); } } } @@ -1123,7 +1260,58 @@ TEST_F(levin_notify, private_local_without_padding) auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; EXPECT_EQ(txs, notification.txs); EXPECT_TRUE(notification._.empty()); - EXPECT_FALSE(notification.dandelionpp_fluff); + EXPECT_TRUE(notification.dandelionpp_fluff); + } + } +} + +TEST_F(levin_notify, private_forward_without_padding) +{ + // private mode always uses fluff but marked as stem + cryptonote::levin::notify notifier = make_notifier(0, false, false); + + for (unsigned count = 0; count < 10; ++count) + add_connection(count % 2 == 0); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + txs[1].resize(200, 'f'); + + ASSERT_EQ(10u, contexts_.size()); + { + auto context = contexts_.begin(); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), events_, cryptonote::relay_method::forward)); + + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + notifier.run_fluff(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + + EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::forward)); + + EXPECT_EQ(0u, context->process_send_queue()); + for (++context; context != contexts_.end(); ++context) + { + const bool is_incoming = ((context - contexts_.begin()) % 2 == 0); + EXPECT_EQ(is_incoming ? 0u : 1u, context->process_send_queue()); + } + + ASSERT_EQ(5u, receiver_.notified_size()); + for (unsigned count = 0; count < 5; ++count) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_TRUE(notification._.empty()); + EXPECT_TRUE(notification.dandelionpp_fluff); } } } @@ -1233,7 +1421,7 @@ TEST_F(levin_notify, private_fluff_with_padding) auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; EXPECT_EQ(txs, notification.txs); EXPECT_FALSE(notification._.empty()); - EXPECT_FALSE(notification.dandelionpp_fluff); + EXPECT_TRUE(notification.dandelionpp_fluff); } } } @@ -1268,7 +1456,7 @@ TEST_F(levin_notify, private_stem_with_padding) io_service_.reset(); ASSERT_LT(0u, io_service_.poll()); - EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff)); + EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem)); EXPECT_EQ(0u, context->process_send_queue()); for (++context; context != contexts_.end(); ++context) @@ -1283,7 +1471,7 @@ TEST_F(levin_notify, private_stem_with_padding) auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; EXPECT_EQ(txs, notification.txs); EXPECT_FALSE(notification._.empty()); - EXPECT_FALSE(notification.dandelionpp_fluff); + EXPECT_TRUE(notification.dandelionpp_fluff); } } } @@ -1333,7 +1521,57 @@ TEST_F(levin_notify, private_local_with_padding) auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; EXPECT_EQ(txs, notification.txs); EXPECT_FALSE(notification._.empty()); - EXPECT_FALSE(notification.dandelionpp_fluff); + EXPECT_TRUE(notification.dandelionpp_fluff); + } + } +} + +TEST_F(levin_notify, private_forward_with_padding) +{ + cryptonote::levin::notify notifier = make_notifier(0, false, true); + + for (unsigned count = 0; count < 10; ++count) + add_connection(count % 2 == 0); + + { + const auto status = notifier.get_status(); + EXPECT_FALSE(status.has_noise); + EXPECT_FALSE(status.connections_filled); + } + notifier.new_out_connection(); + io_service_.poll(); + + std::vector<cryptonote::blobdata> txs(2); + txs[0].resize(100, 'e'); + txs[1].resize(200, 'f'); + + ASSERT_EQ(10u, contexts_.size()); + { + auto context = contexts_.begin(); + EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), events_, cryptonote::relay_method::forward)); + + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + notifier.run_fluff(); + io_service_.reset(); + ASSERT_LT(0u, io_service_.poll()); + + EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::forward)); + + EXPECT_EQ(0u, context->process_send_queue()); + for (++context; context != contexts_.end(); ++context) + { + const bool is_incoming = ((context - contexts_.begin()) % 2 == 0); + EXPECT_EQ(is_incoming ? 0u : 1u, context->process_send_queue()); + } + + ASSERT_EQ(5u, receiver_.notified_size()); + for (unsigned count = 0; count < 5; ++count) + { + auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second; + EXPECT_EQ(txs, notification.txs); + EXPECT_FALSE(notification._.empty()); + EXPECT_TRUE(notification.dandelionpp_fluff); } } } diff --git a/tests/unit_tests/multisig.cpp b/tests/unit_tests/multisig.cpp index 3171a64c1..79775960d 100644 --- a/tests/unit_tests/multisig.cpp +++ b/tests/unit_tests/multisig.cpp @@ -71,7 +71,7 @@ static void make_wallet(unsigned int idx, tools::wallet2 &wallet) try { - wallet.init("", boost::none, boost::asio::ip::tcp::endpoint{}, 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + wallet.init("", boost::none, "", 0, true, epee::net_utils::ssl_support_t::e_ssl_support_disabled); wallet.set_subaddress_lookahead(1, 1); wallet.generate("", "", spendkey, true, false); ASSERT_TRUE(test_addresses[idx].address == wallet.get_account().get_public_address_str(cryptonote::TESTNET)); diff --git a/tests/unit_tests/rolling_median.cpp b/tests/unit_tests/rolling_median.cpp index 730f1e7c5..d415c5b95 100644 --- a/tests/unit_tests/rolling_median.cpp +++ b/tests/unit_tests/rolling_median.cpp @@ -170,6 +170,17 @@ TEST(rolling_median, history_blind) } } +TEST(rolling_median, overflow) +{ + epee::misc_utils::rolling_median_t<uint64_t> m(2); + + uint64_t over_half = static_cast<uint64_t>(3) << static_cast<uint64_t>(62); + m.insert(over_half); + m.insert(over_half); + ASSERT_EQ((over_half + over_half) < over_half, true); + ASSERT_EQ(over_half, m.median()); +} + TEST(rolling_median, size) { epee::misc_utils::rolling_median_t<uint64_t> m(10); diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index ee205e666..e730f6867 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -42,7 +42,7 @@ #include "serialization/json_archive.h" #include "serialization/debug_archive.h" #include "serialization/variant.h" -#include "serialization/vector.h" +#include "serialization/containers.h" #include "serialization/binary_utils.h" #include "wallet/wallet2.h" #include "gtest/gtest.h" diff --git a/tests/unit_tests/tx_proof.cpp b/tests/unit_tests/tx_proof.cpp new file mode 100644 index 000000000..c5d06bc68 --- /dev/null +++ b/tests/unit_tests/tx_proof.cpp @@ -0,0 +1,130 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "gtest/gtest.h" + +#include "crypto/crypto.h" +extern "C" { +#include "crypto/crypto-ops.h" +} +#include "crypto/hash.h" +#include <boost/algorithm/string.hpp> + +static inline unsigned char *operator &(crypto::ec_point &point) { + return &reinterpret_cast<unsigned char &>(point); + } + +static inline unsigned char *operator &(crypto::ec_scalar &scalar) { + return &reinterpret_cast<unsigned char &>(scalar); + } + +TEST(tx_proof, prove_verify_v2) +{ + crypto::secret_key r; + crypto::random32_unbiased(&r); + + // A = aG + // B = bG + crypto::secret_key a,b; + crypto::public_key A,B; + crypto::generate_keys(A, a, a, false); + crypto::generate_keys(B, b, b, false); + + // R_B = rB + crypto::public_key R_B; + ge_p3 B_p3; + ge_frombytes_vartime(&B_p3,&B); + ge_p2 R_B_p2; + ge_scalarmult(&R_B_p2, &unwrap(r), &B_p3); + ge_tobytes(&R_B, &R_B_p2); + + // R_G = rG + crypto::public_key R_G; + ge_frombytes_vartime(&B_p3,&B); + ge_p3 R_G_p3; + ge_scalarmult_base(&R_G_p3, &unwrap(r)); + ge_p3_tobytes(&R_G, &R_G_p3); + + // D = rA + crypto::public_key D; + ge_p3 A_p3; + ge_frombytes_vartime(&A_p3,&A); + ge_p2 D_p2; + ge_scalarmult(&D_p2, &unwrap(r), &A_p3); + ge_tobytes(&D, &D_p2); + + crypto::signature sig; + + // Message data + crypto::hash prefix_hash; + char data[] = "hash input"; + crypto::cn_fast_hash(data,sizeof(data)-1,prefix_hash); + + // Generate/verify valid v1 proof with standard address + crypto::generate_tx_proof_v1(prefix_hash, R_G, A, boost::none, D, r, sig); + ASSERT_TRUE(crypto::check_tx_proof(prefix_hash, R_G, A, boost::none, D, sig, 1)); + + // Generate/verify valid v1 proof with subaddress + crypto::generate_tx_proof_v1(prefix_hash, R_B, A, B, D, r, sig); + ASSERT_TRUE(crypto::check_tx_proof(prefix_hash, R_B, A, B, D, sig, 1)); + + // Generate/verify valid v2 proof with standard address + crypto::generate_tx_proof(prefix_hash, R_G, A, boost::none, D, r, sig); + ASSERT_TRUE(crypto::check_tx_proof(prefix_hash, R_G, A, boost::none, D, sig, 2)); + + // Generate/verify valid v2 proof with subaddress + crypto::generate_tx_proof(prefix_hash, R_B, A, B, D, r, sig); + ASSERT_TRUE(crypto::check_tx_proof(prefix_hash, R_B, A, B, D, sig, 2)); + + // Try to verify valid v2 proofs as v1 proof (bad) + crypto::generate_tx_proof(prefix_hash, R_G, A, boost::none, D, r, sig); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_G, A, boost::none, D, sig, 1)); + crypto::generate_tx_proof(prefix_hash, R_B, A, B, D, r, sig); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_B, A, B, D, sig, 1)); + + // Randomly-distributed test points + crypto::secret_key evil_a, evil_b, evil_d, evil_r; + crypto::public_key evil_A, evil_B, evil_D, evil_R; + crypto::generate_keys(evil_A, evil_a, evil_a, false); + crypto::generate_keys(evil_B, evil_b, evil_b, false); + crypto::generate_keys(evil_D, evil_d, evil_d, false); + crypto::generate_keys(evil_R, evil_r, evil_r, false); + + // Selectively choose bad point in v2 proof (bad) + crypto::generate_tx_proof(prefix_hash, R_B, A, B, D, r, sig); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, evil_R, A, B, D, sig, 2)); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_B, evil_A, B, D, sig, 2)); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_B, A, evil_B, D, sig, 2)); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_B, A, B, evil_D, sig, 2)); + + // Try to verify valid v1 proofs as v2 proof (bad) + crypto::generate_tx_proof_v1(prefix_hash, R_G, A, boost::none, D, r, sig); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_G, A, boost::none, D, sig, 2)); + crypto::generate_tx_proof_v1(prefix_hash, R_B, A, B, D, r, sig); + ASSERT_FALSE(crypto::check_tx_proof(prefix_hash, R_B, A, B, D, sig, 2)); +} diff --git a/tests/unit_tests/varint.cpp b/tests/unit_tests/varint.cpp index ca5af5ad2..a8dee677a 100644 --- a/tests/unit_tests/varint.cpp +++ b/tests/unit_tests/varint.cpp @@ -40,7 +40,7 @@ #include "serialization/json_archive.h" #include "serialization/debug_archive.h" #include "serialization/variant.h" -#include "serialization/vector.h" +#include "serialization/containers.h" #include "serialization/binary_utils.h" #include "gtest/gtest.h" using namespace std; diff --git a/tests/unit_tests/zmq_rpc.cpp b/tests/unit_tests/zmq_rpc.cpp index af1f1608b..59759bed8 100644 --- a/tests/unit_tests/zmq_rpc.cpp +++ b/tests/unit_tests/zmq_rpc.cpp @@ -26,11 +26,25 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include <boost/preprocessor/stringize.hpp> #include <gtest/gtest.h> +#include <rapidjson/document.h> +#include "cryptonote_basic/account.h" +#include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_basic/events.h" +#include "cryptonote_basic/cryptonote_format_utils.h" +#include "json_serialization.h" +#include "net/zmq.h" #include "rpc/message.h" +#include "rpc/zmq_pub.h" +#include "rpc/zmq_server.h" #include "serialization/json_object.h" +#define MASSERT(...) \ + if (!(__VA_ARGS__)) \ + return testing::AssertionFailure() << BOOST_PP_STRINGIZE(__VA_ARGS__) + TEST(ZmqFullMessage, InvalidRequest) { EXPECT_THROW( @@ -53,3 +67,711 @@ TEST(ZmqFullMessage, Request) cryptonote::rpc::FullMessage parsed{request, true}; EXPECT_STREQ("foo", parsed.getRequestType().c_str()); } + +namespace +{ + using published_json = std::pair<std::string, rapidjson::Document>; + + constexpr const char inproc_pub[] = "inproc://dummy_pub"; + + net::zmq::socket create_socket(void* ctx, const char* address) + { + net::zmq::socket sock{zmq_socket(ctx, ZMQ_PAIR)}; + if (!sock) + MONERO_ZMQ_THROW("failed to create socket"); + if (zmq_bind(sock.get(), address) != 0) + MONERO_ZMQ_THROW("socket bind failure"); + return sock; + } + + std::vector<std::string> get_messages(void* socket, int count = -1) + { + std::vector<std::string> out; + for ( ; count || count < 0; --count) + { + expect<std::string> next = net::zmq::receive(socket, (count < 0 ? ZMQ_DONTWAIT : 0)); + if (next == net::zmq::make_error_code(EAGAIN)) + return out; + out.push_back(std::move(*next)); + } + return out; + } + + std::vector<published_json> get_published(void* socket, int count = -1) + { + std::vector<published_json> out; + + const auto messages = get_messages(socket, count); + out.reserve(messages.size()); + + for (const std::string& message : messages) + { + const char* split = std::strchr(message.c_str(), ':'); + if (!split) + throw std::runtime_error{"Invalid ZMQ/Pub message"}; + + out.emplace_back(); + out.back().first = {message.c_str(), split}; + if (out.back().second.Parse(split + 1).HasParseError()) + throw std::runtime_error{"Failed to parse ZMQ/Pub message"}; + } + + return out; + } + + testing::AssertionResult compare_full_txpool(epee::span<const cryptonote::txpool_event> events, const published_json& pub) + { + MASSERT(pub.first == "json-full-txpool_add"); + MASSERT(pub.second.IsArray()); + MASSERT(pub.second.Size() <= events.size()); + + std::size_t i = 0; + for (const cryptonote::txpool_event& event : events) + { + MASSERT(i <= pub.second.Size()); + if (!event.res) + continue; + + cryptonote::transaction tx{}; + cryptonote::json::fromJsonValue(pub.second[i], tx); + + crypto::hash id{}; + MASSERT(cryptonote::get_transaction_hash(event.tx, id)); + MASSERT(cryptonote::get_transaction_hash(tx, id)); + MASSERT(event.tx.hash == tx.hash); + ++i; + } + return testing::AssertionSuccess(); + } + + testing::AssertionResult compare_minimal_txpool(epee::span<const cryptonote::txpool_event> events, const published_json& pub) + { + MASSERT(pub.first == "json-minimal-txpool_add"); + MASSERT(pub.second.IsArray()); + MASSERT(pub.second.Size() <= events.size()); + + std::size_t i = 0; + for (const cryptonote::txpool_event& event : events) + { + MASSERT(i <= pub.second.Size()); + if (!event.res) + continue; + + std::size_t actual_size = 0; + crypto::hash actual_id{}; + + MASSERT(pub.second[i].IsObject()); + GET_FROM_JSON_OBJECT(pub.second[i], actual_id, id); + GET_FROM_JSON_OBJECT(pub.second[i], actual_size, blob_size); + + std::size_t expected_size = 0; + crypto::hash expected_id{}; + MASSERT(cryptonote::get_transaction_hash(event.tx, expected_id, expected_size)); + MASSERT(expected_size == actual_size); + MASSERT(expected_id == actual_id); + ++i; + } + return testing::AssertionSuccess(); + } + + testing::AssertionResult compare_full_block(const epee::span<const cryptonote::block> expected, const published_json& pub) + { + MASSERT(pub.first == "json-full-chain_main"); + MASSERT(pub.second.IsArray()); + + std::vector<cryptonote::block> actual; + cryptonote::json::fromJsonValue(pub.second, actual); + + MASSERT(expected.size() == actual.size()); + + for (std::size_t i = 0; i < expected.size(); ++i) + { + crypto::hash id; + MASSERT(cryptonote::get_block_hash(expected[i], id)); + MASSERT(cryptonote::get_block_hash(actual[i], id)); + MASSERT(expected[i].hash == actual[i].hash); + } + + return testing::AssertionSuccess(); + } + + testing::AssertionResult compare_minimal_block(std::size_t height, const epee::span<const cryptonote::block> expected, const published_json& pub) + { + MASSERT(pub.first == "json-minimal-chain_main"); + MASSERT(pub.second.IsObject()); + MASSERT(!expected.empty()); + + std::size_t actual_height = 0; + crypto::hash actual_id{}; + crypto::hash actual_prev_id{}; + std::vector<crypto::hash> actual_ids{}; + GET_FROM_JSON_OBJECT(pub.second, actual_height, first_height); + GET_FROM_JSON_OBJECT(pub.second, actual_prev_id, first_prev_id); + GET_FROM_JSON_OBJECT(pub.second, actual_ids, ids); + + MASSERT(height == actual_height); + MASSERT(expected[0].prev_id == actual_prev_id); + MASSERT(expected.size() == actual_ids.size()); + + for (std::size_t i = 0; i < expected.size(); ++i) + { + crypto::hash id; + MASSERT(cryptonote::get_block_hash(expected[i], id)); + MASSERT(id == actual_ids[i]); + } + + return testing::AssertionSuccess(); + } + + struct zmq_base : public testing::Test + { + cryptonote::account_base acct; + + zmq_base() + : testing::Test(), acct() + { + acct.generate(); + } + + cryptonote::transaction make_miner_transaction() + { + return test::make_miner_transaction(acct.get_keys().m_account_address); + } + + cryptonote::transaction make_transaction(const std::vector<cryptonote::account_public_address>& destinations) + { + return test::make_transaction(acct.get_keys(), {make_miner_transaction()}, destinations, true, true); + } + + cryptonote::transaction make_transaction() + { + cryptonote::account_base temp_account; + temp_account.generate(); + return make_transaction({temp_account.get_keys().m_account_address}); + } + + cryptonote::block make_block() + { + cryptonote::block block{}; + block.major_version = 1; + block.minor_version = 3; + block.timestamp = 100; + block.prev_id = crypto::rand<crypto::hash>(); + block.nonce = 100; + block.miner_tx = make_miner_transaction(); + return block; + } + }; + + struct zmq_pub : public zmq_base + { + net::zmq::context ctx; + net::zmq::socket relay; + net::zmq::socket dummy_pub; + net::zmq::socket dummy_client; + std::shared_ptr<cryptonote::listener::zmq_pub> pub; + + zmq_pub() + : zmq_base(), + ctx(zmq_init(1)), + relay(create_socket(ctx.get(), cryptonote::listener::zmq_pub::relay_endpoint())), + dummy_pub(create_socket(ctx.get(), inproc_pub)), + dummy_client(zmq_socket(ctx.get(), ZMQ_PAIR)), + pub(std::make_shared<cryptonote::listener::zmq_pub>(ctx.get())) + { + if (!dummy_client) + MONERO_ZMQ_THROW("failed to create socket"); + if (zmq_connect(dummy_client.get(), inproc_pub) != 0) + MONERO_ZMQ_THROW("failed to connect to dummy pub"); + } + + virtual void TearDown() override final + { + EXPECT_EQ(0u, get_messages(relay.get()).size()); + EXPECT_EQ(0u, get_messages(dummy_client.get()).size()); + } + + template<std::size_t N> + bool sub_request(const char (&topic)[N]) + { + return pub->sub_request({topic, N - 1}); + } + }; + + struct dummy_handler final : cryptonote::rpc::RpcHandler + { + dummy_handler() + : cryptonote::rpc::RpcHandler() + {} + + virtual epee::byte_slice handle(std::string&& request) override final + { + throw std::logic_error{"not implemented"}; + } + }; + + struct zmq_server : public zmq_base + { + dummy_handler handler; + cryptonote::rpc::ZmqServer server; + std::shared_ptr<cryptonote::listener::zmq_pub> pub; + net::zmq::socket sub; + + zmq_server() + : zmq_base(), + handler(), + server(handler), + pub(), + sub() + { + void* ctx = server.init_rpc({}, {}); + if (!ctx) + throw std::runtime_error{"init_rpc failure"}; + + const std::string endpoint = inproc_pub; + pub = server.init_pub({std::addressof(endpoint), 1}); + if (!pub) + throw std::runtime_error{"failed to initiaze zmq/pub"}; + + sub.reset(zmq_socket(ctx, ZMQ_SUB)); + if (!sub) + MONERO_ZMQ_THROW("failed to create socket"); + if (zmq_connect(sub.get(), inproc_pub) != 0) + MONERO_ZMQ_THROW("failed to connect to dummy pub"); + + server.run(); + } + + virtual void TearDown() override final + { + EXPECT_EQ(0u, get_messages(sub.get()).size()); + sub.reset(); + pub.reset(); + server.stop(); + } + + template<std::size_t N> + void subscribe(const char (&topic)[N]) + { + if (zmq_setsockopt(sub.get(), ZMQ_SUBSCRIBE, topic, N - 1) != 0) + MONERO_ZMQ_THROW("failed to subscribe"); + } + }; +} + +TEST_F(zmq_pub, InvalidContext) +{ + EXPECT_THROW(cryptonote::listener::zmq_pub{nullptr}, std::logic_error); +} + +TEST_F(zmq_pub, NoBlocking) +{ + EXPECT_FALSE(pub->relay_to_pub(relay.get(), dummy_pub.get())); +} + +TEST_F(zmq_pub, DefaultDrop) +{ + EXPECT_EQ(0u, pub->send_txpool_add({{make_transaction(), {}, true}})); + + const cryptonote::block bl = make_block(); + EXPECT_EQ(0u,pub->send_chain_main(5, {std::addressof(bl), 1})); + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(5, {std::addressof(bl), 1})); +} + +TEST_F(zmq_pub, JsonFullTxpool) +{ + static constexpr const char topic[] = "\1json-full-txpool_add"; + + ASSERT_TRUE(sub_request(topic)); + + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + EXPECT_NO_THROW(pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_EQ(1u, pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); +} + +TEST_F(zmq_pub, JsonMinimalTxpool) +{ + static constexpr const char topic[] = "\1json-minimal-txpool_add"; + + ASSERT_TRUE(sub_request(topic)); + + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + EXPECT_NO_THROW(pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_EQ(1u, pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); +} + +TEST_F(zmq_pub, JsonFullChain) +{ + static constexpr const char topic[] = "\1json-full-chain_main"; + + ASSERT_TRUE(sub_request(topic)); + + const std::array<cryptonote::block, 2> blocks{{make_block(), make_block()}}; + + EXPECT_EQ(1u, pub->send_chain_main(100, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_block(epee::to_span(blocks), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(533, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_block(epee::to_span(blocks), pubs.front())); +} + +TEST_F(zmq_pub, JsonMinimalChain) +{ + static constexpr const char topic[] = "\1json-minimal-chain_main"; + + ASSERT_TRUE(sub_request(topic)); + + const std::array<cryptonote::block, 2> blocks{{make_block(), make_block()}}; + + EXPECT_EQ(1u, pub->send_chain_main(100, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_block(100, epee::to_span(blocks), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(533, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_block(533, epee::to_span(blocks), pubs.front())); +} + +TEST_F(zmq_pub, JsonFullAll) +{ + static constexpr const char topic[] = "\1json-full"; + + ASSERT_TRUE(sub_request(topic)); + { + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + EXPECT_EQ(1u, pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_NO_THROW(pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + } + { + const std::array<cryptonote::block, 2> blocks{{make_block(), make_block()}}; + + EXPECT_EQ(1u, pub->send_chain_main(100, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_block(epee::to_span(blocks), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(533, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_full_block(epee::to_span(blocks), pubs.front())); + } +} + +TEST_F(zmq_pub, JsonMinimalAll) +{ + static constexpr const char topic[] = "\1json-minimal"; + + ASSERT_TRUE(sub_request(topic)); + + { + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + EXPECT_EQ(1u, pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_NO_THROW(pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + + events.at(0).res = false; + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + } + { + const std::array<cryptonote::block, 2> blocks{{make_block(), make_block()}}; + + EXPECT_EQ(1u, pub->send_chain_main(100, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_block(100, epee::to_span(blocks), pubs.front())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(533, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(1u, pubs.size()); + ASSERT_LE(1u, pubs.size()); + EXPECT_TRUE(compare_minimal_block(533, epee::to_span(blocks), pubs.front())); + } +} + +TEST_F(zmq_pub, JsonAll) +{ + static constexpr const char topic[] = "\1json"; + + ASSERT_TRUE(sub_request(topic)); + + { + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + EXPECT_EQ(1u, pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.back())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.back())); + + events.at(0).res = false; + EXPECT_EQ(1u, pub->send_txpool_add(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.back())); + + events.at(0).res = false; + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(events)); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_full_txpool(epee::to_span(events), pubs.front())); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.back())); + } + { + const std::array<cryptonote::block, 1> blocks{{make_block()}}; + + EXPECT_EQ(2u, pub->send_chain_main(100, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + auto pubs = get_published(dummy_client.get()); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_full_block(epee::to_span(blocks), pubs.front())); + EXPECT_TRUE(compare_minimal_block(100, epee::to_span(blocks), pubs.back())); + + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(533, epee::to_span(blocks))); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + EXPECT_TRUE(pub->relay_to_pub(relay.get(), dummy_pub.get())); + + pubs = get_published(dummy_client.get()); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_full_block(epee::to_span(blocks), pubs.front())); + EXPECT_TRUE(compare_minimal_block(533, epee::to_span(blocks), pubs.back())); + } +} + +TEST_F(zmq_pub, JsonChainWeakPtrSkip) +{ + static constexpr const char topic[] = "\1json"; + + ASSERT_TRUE(sub_request(topic)); + + const std::array<cryptonote::block, 1> blocks{{make_block()}}; + + pub.reset(); + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::chain_main{pub}(533, epee::to_span(blocks))); +} + +TEST_F(zmq_pub, JsonTxpoolWeakPtrSkip) +{ + static constexpr const char topic[] = "\1json"; + + ASSERT_TRUE(sub_request(topic)); + + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + pub.reset(); + EXPECT_NO_THROW(cryptonote::listener::zmq_pub::txpool_add{pub}(std::move(events))); +} + +TEST_F(zmq_server, pub) +{ + subscribe("json-minimal"); + + std::vector<cryptonote::txpool_event> events + { + {make_transaction(), {}, true}, {make_transaction(), {}, true} + }; + + const std::array<cryptonote::block, 1> blocks{{make_block()}}; + + ASSERT_EQ(1u, pub->send_txpool_add(events)); + ASSERT_EQ(1u, pub->send_chain_main(200, epee::to_span(blocks))); + + auto pubs = get_published(sub.get(), 2); + EXPECT_EQ(2u, pubs.size()); + ASSERT_LE(2u, pubs.size()); + EXPECT_TRUE(compare_minimal_txpool(epee::to_span(events), pubs.front())); + EXPECT_TRUE(compare_minimal_block(200, epee::to_span(blocks), pubs.back())); +} diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 074f8de37..435bc93f8 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -465,7 +465,7 @@ class Daemon(object): def in_peers(self, in_peers): in_peers = { - 'client': client, + 'in_peers': in_peers, } return self.rpc.send_request('/in_peers', in_peers) @@ -554,6 +554,16 @@ class Daemon(object): } return self.rpc.send_json_rpc_request(flush_cache) + def sync_txpool(self): + sync_txpool = { + 'method': 'sync_txpool', + 'params': { + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(sync_txpool) + def rpc_access_info(self, client): rpc_access_info = { 'method': 'rpc_access_info', |