diff options
90 files changed, 1982 insertions, 535 deletions
diff --git a/.gitignore b/.gitignore index d321aa43d..0ece7cb75 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,9 @@ local.properties # PDT-specific .buildpath +# Netbeans-specific +nbproject + # sbteclipse plugin .target @@ -103,4 +106,4 @@ local.properties .texlipse .idea/ -/testnet
\ No newline at end of file +/testnet diff --git a/.travis.yml b/.travis.yml index d83675869..07b92aaff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ env: - SDK_URL=https://bitcoincore.org/depends-sources/sdks - DOCKER_PACKAGES="build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" matrix: +# RISCV 64bit + - HOST=riscv64-linux-gnu PACKAGES="python3 gperf g++-riscv64-linux-gnu" # ARM v7 - HOST=arm-linux-gnueabihf PACKAGES="python3 gperf g++-arm-linux-gnueabihf" # ARM v8 diff --git a/CMakeLists.txt b/CMakeLists.txt index b150a8998..60fcf130e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -651,7 +651,7 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing") # if those don't work for your compiler, single it out where appropriate - if(CMAKE_BUILD_TYPE STREQUAL "Release") + if(CMAKE_BUILD_TYPE STREQUAL "Release" AND NOT OPENBSD) set(C_SECURITY_FLAGS "${C_SECURITY_FLAGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1") set(CXX_SECURITY_FLAGS "${CXX_SECURITY_FLAGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1") endif() @@ -663,7 +663,7 @@ else() add_cxx_flag_if_supported(-Wformat-security CXX_SECURITY_FLAGS) # -fstack-protector - if (NOT WIN32) + if (NOT WIN32 AND NOT OPENBSD) add_c_flag_if_supported(-fstack-protector C_SECURITY_FLAGS) add_cxx_flag_if_supported(-fstack-protector CXX_SECURITY_FLAGS) add_c_flag_if_supported(-fstack-protector-strong C_SECURITY_FLAGS) @@ -671,7 +671,7 @@ else() endif() # New in GCC 8.2 - if (NOT WIN32) + if (NOT WIN32 AND NOT OPENBSD) add_c_flag_if_supported(-fcf-protection=full C_SECURITY_FLAGS) add_cxx_flag_if_supported(-fcf-protection=full CXX_SECURITY_FLAGS) add_c_flag_if_supported(-fstack-clash-protection C_SECURITY_FLAGS) @@ -1003,6 +1003,7 @@ endif() add_subdirectory(contrib) add_subdirectory(src) +find_package(PythonInterp) if(BUILD_TESTS) message(STATUS "Building tests") add_subdirectory(tests) @@ -1047,5 +1048,3 @@ option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) - -find_package(PythonInterp) @@ -10,7 +10,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - [Research](#research) - [Announcements](#announcements) - [Translations](#translations) - - [Build](#build) + - [Build Status](#build-status) - [IMPORTANT](#important) - [Coverage](#coverage) - [Introduction](#introduction) @@ -22,6 +22,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - [Release staging schedule and protocol](#release-staging-schedule-and-protocol) - [Compiling Monero from source](#compiling-monero-from-source) - [Dependencies](#dependencies) + - [Known issues](#known-issues) ## Development resources @@ -55,7 +56,7 @@ The CLI wallet is available in different languages. If you want to help translat If you need help/support/info about translations, contact the localization workgroup. You can find the complete list of contacts on the repository of the workgroup: [monero-translations](https://github.com/monero-ecosystem/monero-translations#contacts). -## Build +## Build Status ### IMPORTANT @@ -463,100 +464,10 @@ We expect to add Monero into the ports tree in the near future, which will aid i ### On OpenBSD: -#### OpenBSD < 6.2 - -This has been tested on OpenBSD 5.8. - -You will need to add a few packages to your system. `pkg_add db cmake gcc gcc-libs g++ gtest`. - -The doxygen and graphviz packages are optional and require the xbase set. - -The Boost package has a bug that will prevent librpc.a from building correctly. In order to fix this, you will have to Build boost yourself from scratch. Follow the directions here (under "Building Boost"): -https://github.com/bitcoin/bitcoin/blob/master/doc/build-openbsd.md - -You will have to add the serialization, date_time, and regex modules to Boost when building as they are needed by Monero. - -To build: `env CC=egcc CXX=eg++ CPP=ecpp DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/path/to/the/boost/you/built make release-static-64` - -#### OpenBSD 6.2 and 6.3 - -You will need to add a few packages to your system. `pkg_add cmake zeromq libiconv`. - -The doxygen and graphviz packages are optional and require the xbase set. - - -Build the Boost library using clang. This guide is derived from: https://github.com/bitcoin/bitcoin/blob/master/doc/build-openbsd.md - -We assume you are compiling with a non-root user and you have `doas` enabled. - -Note: do not use the boost package provided by OpenBSD, as we are installing boost to `/usr/local`. - -```bash -# Create boost building directory -mkdir ~/boost -cd ~/boost - -# Fetch boost source -ftp -o boost_1_64_0.tar.bz2 https://netcologne.dl.sourceforge.net/project/boost/boost/1.64.0/boost_1_64_0.tar.bz2 - -# MUST output: (SHA256) boost_1_64_0.tar.bz2: OK -echo "7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 boost_1_64_0.tar.bz2" | sha256 -c -tar xfj boost_1_64_0.tar.bz2 - -# Fetch and apply boost patches, required for OpenBSD -ftp -o boost_test_impl_execution_monitor_ipp.patch https://raw.githubusercontent.com/openbsd/ports/bee9e6df517077a7269ff0dfd57995f5c6a10379/devel/boost/patches/patch-boost_test_impl_execution_monitor_ipp -ftp -o boost_config_platform_bsd_hpp.patch https://raw.githubusercontent.com/openbsd/ports/90658284fb786f5a60dd9d6e8d14500c167bdaa0/devel/boost/patches/patch-boost_config_platform_bsd_hpp - -# MUST output: (SHA256) boost_config_platform_bsd_hpp.patch: OK -echo "1f5e59d1154f16ee1e0cc169395f30d5e7d22a5bd9f86358f738b0ccaea5e51d boost_config_platform_bsd_hpp.patch" | sha256 -c -# MUST output: (SHA256) boost_test_impl_execution_monitor_ipp.patch: OK -echo "30cec182a1437d40c3e0bd9a866ab5ddc1400a56185b7e671bb3782634ed0206 boost_test_impl_execution_monitor_ipp.patch" | sha256 -c - -cd boost_1_64_0 -patch -p0 < ../boost_test_impl_execution_monitor_ipp.patch -patch -p0 < ../boost_config_platform_bsd_hpp.patch - -# Start building boost -echo 'using clang : : c++ : <cxxflags>"-fvisibility=hidden -fPIC" <linkflags>"" <archiver>"ar" <striper>"strip" <ranlib>"ranlib" <rc>"" : ;' > user-config.jam -./bootstrap.sh --without-icu --with-libraries=chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale --with-toolset=clang -./b2 toolset=clang cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" -sICONV_PATH=/usr/local -doas ./b2 -d0 runtime-link=shared threadapi=pthread threading=multi link=static variant=release --layout=tagged --build-type=complete --user-config=user-config.jam -sNO_BZIP2=1 -sICONV_PATH=/usr/local --prefix=/usr/local install -``` - -Build the cppzmq bindings. - -We assume you are compiling with a non-root user and you have `doas` enabled. - -```bash -# Create cppzmq building directory -mkdir ~/cppzmq -cd ~/cppzmq - -# Fetch cppzmq source -ftp -o cppzmq-4.2.3.tar.gz https://github.com/zeromq/cppzmq/archive/v4.2.3.tar.gz - -# MUST output: (SHA256) cppzmq-4.2.3.tar.gz: OK -echo "3e6b57bf49115f4ae893b1ff7848ead7267013087dc7be1ab27636a97144d373 cppzmq-4.2.3.tar.gz" | sha256 -c -tar xfz cppzmq-4.2.3.tar.gz - -# Start building cppzmq -cd cppzmq-4.2.3 -mkdir build -cd build -cmake .. -doas make install -``` - -Build monero: -```bash -env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local make release-static -``` - -#### OpenBSD >= 6.4 - You will need to add a few packages to your system. `pkg_add cmake gmake zeromq cppzmq libiconv boost`. -The doxygen and graphviz packages are optional and require the xbase set. +The `doxygen` and `graphviz` packages are optional and require the xbase set. +Running the test suite also requires `py-requests` package. Build monero: `env DEVELOPER_LOCAL_TOOLS=1 BOOST_ROOT=/usr/local gmake release-static` @@ -624,6 +535,8 @@ You can also cross-compile static binaries on Linux for Windows and macOS with t * Requires: `g++-arm-linux-gnueabihf` * ```make depends target=aarch64-linux-gnu``` for armv8 binaries. * Requires: `g++-aarch64-linux-gnu` +* ```make depends target=riscv64-linux-gnu``` for RISC V 64 bit binaries. + * Requires: `g++-riscv64-linux-gnu` The required packages are the names for each toolchain on apt. Depending on your distro, they may have different names. @@ -804,6 +717,12 @@ gdb /path/to/monerod /path/to/dumpfile` Print the stack trace with `bt` + * If a program crashed and cores are managed by systemd, the following can also get a stack trace for that crash: + +```bash +coredumpctl -1 gdb +``` + #### To run monero within gdb: Type `gdb /path/to/monerod` @@ -845,3 +764,20 @@ The output of `mdb_stat -ea <path to blockchain dir>` will indicate inconsistenc The output of `mdb_dump -s blocks <path to blockchain dir>` and `mdb_dump -s block_info <path to blockchain dir>` is useful for indicating whether blocks and block_info contain the same keys. These records are dumped as hex data, where the first line is the key and the second line is the data. + +# Known Issues + +## Protocols + +### Socket-based + +Because of the nature of the socket-based protocols that drive monero, certain protocol weaknesses are somewhat unavoidable at this time. While these weaknesses can theoretically be fully mitigated, the effort required (the means) may not justify the ends. As such, please consider taking the following precautions if you are a monero node operator: + +- Run `monerod` on a "secured" machine. If operational security is not your forte, at a very minimum, have a dedicated a computer running `monerod` and **do not** browse the web, use email clients, or use any other potentially harmful apps on your `monerod` machine. **Do not click links or load URL/MUA content on the same machine**. Doing so may potentially exploit weaknesses in commands which accept "localhost" and "127.0.0.1". +- If you plan on hosting a public "remote" node, start `monerod` with `--restricted-rpc`. This is a must. + +### Blockchain-based + +Certain blockchain "features" can be considered "bugs" if misused correctly. Consequently, please consider the following: + +- When receiving monero, be aware that it may be locked for an arbitrary time if the sender elected to, preventing you from spending that monero until the lock time expires. You may want to hold off acting upon such a transaction until the unlock time lapses. To get a sense of that time, you can consider the remaining blocktime until unlock as seen in the `show_transfers` command. diff --git a/cmake/CheckTrezor.cmake b/cmake/CheckTrezor.cmake index 6aabdda36..d2fe60d92 100644 --- a/cmake/CheckTrezor.cmake +++ b/cmake/CheckTrezor.cmake @@ -159,7 +159,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T set(TREZOR_LIBUSB_LIBRARIES "") if(LibUSB_COMPILE_TEST_PASSED) - list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES}) + list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES} ${LIBUSB_DEP_LINKER}) message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") endif() @@ -174,7 +174,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T if (TREZOR_LIBUSB_LIBRARIES) list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES}) - string(APPEND TREZOR_DEP_LINKER " -lusb-1.0") + string(APPEND TREZOR_DEP_LINKER " -lusb-1.0 ${LIBUSB_DEP_LINKER}") endif() endif() endif() diff --git a/cmake/FindLibUSB.cmake b/cmake/FindLibUSB.cmake index c7e09d4c0..79d8fba75 100644 --- a/cmake/FindLibUSB.cmake +++ b/cmake/FindLibUSB.cmake @@ -119,6 +119,12 @@ if ( LibUSB_FOUND ) find_library(IOKIT IOKit) list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${IOKIT}) list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${COREFOUNDATION}) + + if(STATIC) + find_library(OBJC objc.a) + set(LIBUSB_DEP_LINKER ${OBJC}) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LIBUSB_DEP_LINKER}) + endif() endif() endif() if (WIN32) diff --git a/contrib/depends/README.md b/contrib/depends/README.md index 6a15055b4..10866acbe 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -33,6 +33,7 @@ Common `host-platform-triplets` for cross compilation are: - `x86_64-apple-darwin11` for MacOSX - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit +- `riscv64-linux-gnu` for Linux RISCV 64 bit No other options are needed, the paths are automatically configured. diff --git a/contrib/depends/packages/icu4c.mk b/contrib/depends/packages/icu4c.mk index dc089bbbe..58ae637b0 100644 --- a/contrib/depends/packages/icu4c.mk +++ b/contrib/depends/packages/icu4c.mk @@ -1,8 +1,8 @@ package=icu4c -$(package)_version=55.1 -$(package)_download_path=https://github.com/TheCharlatan/icu4c/archive -$(package)_file_name=55.1.tar.gz -$(package)_sha256_hash=1f912c54035533fb4268809701d65c7468d00e292efbc31e6444908450cc46ef +$(package)_version=55.2 +$(package)_download_path=https://github.com/unicode-org/icu/releases/download/release-55-2/ +$(package)_file_name=$(package)-55_2-src.tgz +$(package)_sha256_hash=eda2aa9f9c787748a2e2d310590720ca8bcc6252adf6b4cfb03b65bef9d66759 $(package)_patches=icu-001-dont-build-static-dynamic-twice.patch define $(package)_set_vars diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index ea4902170..6fbcc3466 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -12,6 +12,10 @@ define $(package)_set_vars $(package)_config_opts_linux=--with-pic endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index e39dc1d04..8d08900f5 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -40,6 +40,7 @@ $(package)_config_opts_x86_64_linux=linux-x86_64 $(package)_config_opts_i686_linux=linux-generic32 $(package)_config_opts_arm_linux=linux-generic32 $(package)_config_opts_aarch64_linux=linux-generic64 +$(package)_config_opts_riscv64_linux=linux-generic64 $(package)_config_opts_mipsel_linux=linux-generic32 $(package)_config_opts_mips_linux=linux-generic32 $(package)_config_opts_powerpc_linux=linux-generic32 diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 562f4f7d6..1e5a74670 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -4,13 +4,17 @@ native_packages := native_ccache native_protobuf darwin_native_packages = native_biplist native_ds_store native_mac_alias darwin_packages = sodium-darwin ncurses readline -linux_packages = eudev ncurses readline unwind sodium +linux_packages = eudev ncurses readline sodium qt_packages = qt ifeq ($(build_tests),ON) packages += gtest endif +ifneq ($(host_arch),riscv64) + packages += unwind +endif + ifeq ($(host_os),mingw32) packages += icu4c packages += sodium diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index ee0407a5e..b748f5c55 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -98,6 +98,11 @@ elseif(ARCHITECTURE STREQUAL "aarch64") set(BUILD_64 ON) endif() +if(ARCHITECTURE STREQUAL "riscv64") + set(NO_AES ON) + set(ARCH "rv64imafdc") +endif() + if(ARCHITECTURE STREQUAL "i686" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") SET(LINUX_32 ON) SET(ARCH_ID "i386") diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index c1aa0fe5f..b38ab5399 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -227,8 +227,12 @@ namespace net_utils std::map<std::string, t_connection_type> server_type_map; void create_server_type_map(); - bool init_server(uint32_t port, const std::string address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); - bool init_server(const std::string port, const std::string& address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(uint32_t port, const std::string& address = "0.0.0.0", + uint32_t port_ipv6 = 0, const std::string& address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true, + ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(const std::string port, const std::string& address = "0.0.0.0", + const std::string port_ipv6 = "", const std::string address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true, + ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); /// Run the server's io_service loop. bool run_server(size_t threads_count, bool wait = true, const boost::thread::attributes& attrs = boost::thread::attributes()); @@ -269,6 +273,7 @@ namespace net_utils } int get_binded_port(){return m_port;} + int get_binded_port_ipv6(){return m_port_ipv6;} long get_connections_count() const { @@ -339,7 +344,9 @@ namespace net_utils /// Run the server's io_service loop. bool worker_thread(); /// Handle completion of an asynchronous accept operation. - void handle_accept(const boost::system::error_code& e); + void handle_accept_ipv4(const boost::system::error_code& e); + void handle_accept_ipv6(const boost::system::error_code& e); + void handle_accept(const boost::system::error_code& e, bool ipv6 = false); bool is_thread_worker(); @@ -360,11 +367,16 @@ namespace net_utils /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; + boost::asio::ip::tcp::acceptor acceptor_ipv6; epee::net_utils::network_address default_remote; std::atomic<bool> m_stop_signal_sent; uint32_t m_port; + uint32_t m_port_ipv6; std::string m_address; + std::string m_address_ipv6; + bool m_use_ipv6; + bool m_require_ipv4; std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; std::vector<boost::shared_ptr<boost::thread> > m_threads; @@ -376,6 +388,8 @@ namespace net_utils /// The next connection to be accepted connection_ptr new_connection_; + connection_ptr new_connection_ipv6; + boost::mutex connections_mutex; std::set<connection_ptr> connections_; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 0721366aa..19e9c9af9 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -145,10 +145,18 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::system::error_code ec; auto remote_ep = socket().remote_endpoint(ec); CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value()); - CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4(), false, "IPv6 not supported here"); + CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4() || remote_ep.address().is_v6(), false, "only IPv4 and IPv6 supported here"); - const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())}; - return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + if (remote_ep.address().is_v4()) + { + const unsigned long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); + return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + } + else + { + const auto ip_{remote_ep.address().to_v6()}; + return start(is_income, is_multithreaded, ipv6_network_address{ip_, remote_ep.port()}); + } CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false); } //--------------------------------------------------------------------------------- @@ -904,12 +912,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_io_service_local_instance(new worker()), io_service_(m_io_service_local_instance->io_service), acceptor_(io_service_), + acceptor_ipv6(io_service_), default_remote(), m_stop_signal_sent(false), m_port(0), m_threads_count(0), m_thread_index(0), m_connection_type( connection_type ), - new_connection_() + new_connection_(), + new_connection_ipv6() { create_server_type_map(); m_thread_name_prefix = "NET"; @@ -920,12 +930,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_state(boost::make_shared<typename connection<t_protocol_handler>::shared_state>()), io_service_(extarnal_io_service), acceptor_(io_service_), + acceptor_ipv6(io_service_), default_remote(), m_stop_signal_sent(false), m_port(0), m_threads_count(0), m_thread_index(0), m_connection_type(connection_type), - new_connection_() + new_connection_(), + new_connection_ipv6() { create_server_type_map(); m_thread_name_prefix = "NET"; @@ -947,29 +959,92 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address, ssl_options_t ssl_options) + bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string& address, + uint32_t port_ipv6, const std::string& address_ipv6, bool use_ipv6, bool require_ipv4, + ssl_options_t ssl_options) { TRY_ENTRY(); m_stop_signal_sent = false; m_port = port; + m_port_ipv6 = port_ipv6; m_address = address; + m_address_ipv6 = address_ipv6; + m_use_ipv6 = use_ipv6; + m_require_ipv4 = require_ipv4; + if (ssl_options) m_state->configure_ssl(std::move(ssl_options)); - // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); - acceptor_.open(endpoint.protocol()); - acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); - acceptor_.bind(endpoint); - acceptor_.listen(); - boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); - m_port = binded_endpoint.port(); - MDEBUG("start accept"); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + + std::string ipv4_failed = ""; + std::string ipv6_failed = ""; + try + { + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_.open(endpoint.protocol()); + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_.bind(endpoint); + acceptor_.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); + m_port = binded_endpoint.port(); + MDEBUG("start accept (IPv4)"); + new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); + acceptor_.async_accept(new_connection_->socket(), + boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4, this, + boost::asio::placeholders::error)); + } + catch (const std::exception &e) + { + ipv4_failed = e.what(); + } + + if (ipv4_failed != "") + { + MERROR("Failed to bind IPv4: " << ipv4_failed); + if (require_ipv4) + { + throw std::runtime_error("Failed to bind IPv4 (set to required)"); + } + } + + if (use_ipv6) + { + try + { + if (port_ipv6 == 0) port_ipv6 = port; // default arg means bind to same port as ipv4 + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address_ipv6, boost::lexical_cast<std::string>(port_ipv6), boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_ipv6.open(endpoint.protocol()); + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + acceptor_ipv6.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_ipv6.set_option(boost::asio::ip::v6_only(true)); + acceptor_ipv6.bind(endpoint); + acceptor_ipv6.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_ipv6.local_endpoint(); + m_port_ipv6 = binded_endpoint.port(); + MDEBUG("start accept (IPv6)"); + new_connection_ipv6.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); + acceptor_ipv6.async_accept(new_connection_ipv6->socket(), + boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6, this, + boost::asio::placeholders::error)); + } + catch (const std::exception &e) + { + ipv6_failed = e.what(); + } + } + + if (use_ipv6 && ipv6_failed != "") + { + MERROR("Failed to bind IPv6: " << ipv6_failed); + if (ipv4_failed != "") + { + throw std::runtime_error("Failed to bind IPv4 and IPv6"); + } + } return true; } @@ -988,15 +1063,23 @@ PRAGMA_WARNING_DISABLE_VS(4355) PUSH_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) template<class t_protocol_handler> - bool boosted_tcp_server<t_protocol_handler>::init_server(const std::string port, const std::string& address, ssl_options_t ssl_options) + bool boosted_tcp_server<t_protocol_handler>::init_server(const std::string port, const std::string& address, + const std::string port_ipv6, const std::string address_ipv6, bool use_ipv6, bool require_ipv4, + ssl_options_t ssl_options) { uint32_t p = 0; + uint32_t p_ipv6 = 0; if (port.size() && !string_tools::get_xtype_from_string(p, port)) { MERROR("Failed to convert port no = " << port); return false; } - return this->init_server(p, address, std::move(ssl_options)); + + if (port_ipv6.size() && !string_tools::get_xtype_from_string(p_ipv6, port_ipv6)) { + MERROR("Failed to convert port no = " << port_ipv6); + return false; + } + return this->init_server(p, address, p_ipv6, address_ipv6, use_ipv6, require_ipv4, std::move(ssl_options)); } POP_WARNINGS //--------------------------------------------------------------------------------- @@ -1088,7 +1171,7 @@ POP_WARNINGS { //some problems with the listening socket ?.. _dbg1("Net service stopped without stop request, restarting..."); - if(!this->init_server(m_port, m_address)) + if(!this->init_server(m_port, m_address, m_port_ipv6, m_address_ipv6, m_use_ipv6, m_require_ipv4)) { _dbg1("Reiniting service failed, exit."); return false; @@ -1154,29 +1237,52 @@ POP_WARNINGS } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e) + void boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4(const boost::system::error_code& e) + { + this->handle_accept(e, false); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6(const boost::system::error_code& e) + { + this->handle_accept(e, true); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e, bool ipv6) { MDEBUG("handle_accept"); + + boost::asio::ip::tcp::acceptor* current_acceptor = &acceptor_; + connection_ptr* current_new_connection = &new_connection_; + auto accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4; + if (ipv6) + { + current_acceptor = &acceptor_ipv6; + current_new_connection = &new_connection_ipv6; + accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6; + } + try { if (!e) { - if (m_connection_type == e_connection_type_RPC) { - const char *ssl_message = "unknown"; - switch (new_connection_->get_ssl_support()) - { - case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; - case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; - case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; - } - MDEBUG("New server for RPC connections, SSL " << ssl_message); - new_connection_->setRpcStation(); // hopefully this is not needed actually - } - connection_ptr conn(std::move(new_connection_)); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support())); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + if (m_connection_type == e_connection_type_RPC) { + const char *ssl_message = "unknown"; + switch ((*current_new_connection)->get_ssl_support()) + { + case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; + } + MDEBUG("New server for RPC connections, SSL " << ssl_message); + (*current_new_connection)->setRpcStation(); // hopefully this is not needed actually + } + connection_ptr conn(std::move((*current_new_connection))); + (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support())); + current_acceptor->async_accept((*current_new_connection)->socket(), + boost::bind(accept_function_pointer, this, + boost::asio::placeholders::error)); boost::asio::socket_base::keep_alive opt(true); conn->socket().set_option(opt); @@ -1208,10 +1314,10 @@ POP_WARNINGS assert(m_state != nullptr); // always set in constructor _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_state->sock_count); misc_utils::sleep_no_w(100); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, new_connection_->get_ssl_support())); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, (*current_new_connection)->get_ssl_support())); + current_acceptor->async_accept((*current_new_connection)->socket(), + boost::bind(accept_function_pointer, this, + boost::asio::placeholders::error)); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -1345,23 +1451,84 @@ POP_WARNINGS epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - ////////////////////////////////////////////////////////////////////////// + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::system::error_code resolve_error; + boost::asio::ip::tcp::resolver::iterator iterator; + try + { + //resolving ipv4 address as ipv6 throws, catch here and move on + iterator = resolver.resolve(query, resolve_error); + } + catch (const boost::system::system_error& e) + { + if (!m_use_ipv6 || (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again)) + { + throw; + } + try_ipv6 = true; + } + catch (...) + { + throw; + } + + std::string bind_ip_to_use; + boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - _erro("Failed to resolve " << adr); - return false; + if (!m_use_ipv6) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + try_ipv6 = true; + MINFO("Resolving address as IPv4 failed, trying IPv6"); + } + } + else + { + bind_ip_to_use = bind_ip; } - ////////////////////////////////////////////////////////////////////////// + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + + iterator = resolver.resolve(query6, resolve_error); + + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + if (bind_ip == "0.0.0.0") + { + bind_ip_to_use = "::"; + } + else + { + bind_ip_to_use = ""; + } + + } + + } + + LOG_ERROR("Trying connect to " << adr << ":" << port << ", bind_ip = " << bind_ip_to_use); //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, ssl_support); + auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout, ssl_support); if (try_connect_result == CONNECT_FAILURE) return false; if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect && try_connect_result == CONNECT_NO_SSL) @@ -1369,7 +1536,7 @@ POP_WARNINGS // we connected, but could not connect with SSL, try without MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); new_connection_l->disable_ssl(); - try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); if (try_connect_result != CONNECT_SUCCESS) return false; } @@ -1409,17 +1576,59 @@ POP_WARNINGS epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - ////////////////////////////////////////////////////////////////////////// + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::system::error_code resolve_error; + boost::asio::ip::tcp::resolver::iterator iterator; + try + { + //resolving ipv4 address as ipv6 throws, catch here and move on + iterator = resolver.resolve(query, resolve_error); + } + catch (const boost::system::system_error& e) + { + if (!m_use_ipv6 || (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again)) + { + throw; + } + try_ipv6 = true; + } + catch (...) + { + throw; + } + boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - _erro("Failed to resolve " << adr); - return false; + if (!try_ipv6) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + MINFO("Resolving address as IPv4 failed, trying IPv6"); + } } - ////////////////////////////////////////////////////////////////////////// + + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + + iterator = resolver.resolve(query6, resolve_error); + + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } + } + + boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); sock_.open(remote_endpoint.protocol()); diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index ae8e43477..790d0f3b1 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -577,6 +577,10 @@ namespace net_utils if (query_info.m_http_method != http::http_method_options) { res = handle_request(query_info, response); + if (response.m_response_code == 500) + { + m_want_close = true; // close on all "Internal server error"s + } } else { diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 4b2053091..07ed8157b 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -71,7 +71,7 @@ MINFO(m_conn_context << "calling " << s_pattern); \ if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ { \ - LOG_ERROR("Failed to " << #callback_f << "()"); \ + MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ @@ -99,7 +99,7 @@ MINFO(m_conn_context << "calling " << s_pattern); \ if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ { \ - LOG_ERROR("Failed to " << #callback_f << "()"); \ + MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ diff --git a/contrib/epee/include/net/http_server_impl_base.h b/contrib/epee/include/net/http_server_impl_base.h index fc2dcbf67..6cd19f17b 100644 --- a/contrib/epee/include/net/http_server_impl_base.h +++ b/contrib/epee/include/net/http_server_impl_base.h @@ -57,6 +57,7 @@ namespace epee {} bool init(std::function<void(size_t, uint8_t*)> rng, const std::string& bind_port = "0", const std::string& bind_ip = "0.0.0.0", + const std::string& bind_ipv6_address = "::", bool use_ipv6 = false, bool require_ipv4 = true, std::vector<std::string> access_control_origins = std::vector<std::string>(), boost::optional<net_utils::http::login> user = boost::none, net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect) @@ -75,8 +76,12 @@ namespace epee m_net_server.get_config_object().m_user = std::move(user); - MGINFO("Binding on " << bind_ip << ":" << bind_port); - bool res = m_net_server.init_server(bind_port, bind_ip, std::move(ssl_options)); + MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port); + if (use_ipv6) + { + MGINFO("Binding on " << bind_ipv6_address << " (IPv6):" << bind_port); + } + bool res = m_net_server.init_server(bind_port, bind_ip, bind_port, bind_ipv6_address, use_ipv6, require_ipv4, std::move(ssl_options)); if(!res) { LOG_ERROR("Failed to bind server"); diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 116b3ace1..8d7ffb2c2 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -99,6 +99,8 @@ public: template<class callback_t> bool for_connection(const boost::uuids::uuid &connection_id, const callback_t &cb); size_t get_connections_count(); + size_t get_out_connections_count(); + size_t get_in_connections_count(); void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL); async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) @@ -882,6 +884,28 @@ size_t async_protocol_handler_config<t_connection_context>::get_connections_coun } //------------------------------------------------------------------------------------------ template<class t_connection_context> +size_t async_protocol_handler_config<t_connection_context>::get_out_connections_count() +{ + CRITICAL_REGION_LOCAL(m_connects_lock); + size_t count = 0; + for (const auto &c: m_connects) + if (!c.second->m_connection_context.m_is_income) + ++count; + return count; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> +size_t async_protocol_handler_config<t_connection_context>::get_in_connections_count() +{ + CRITICAL_REGION_LOCAL(m_connects_lock); + size_t count = 0; + for (const auto &c: m_connects) + if (c.second->m_connection_context.m_is_income) + ++count; + return count; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*)) { if (m_pcommands_handler && m_pcommands_handler_destroy) diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index 52c5855b9..7523f9d81 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -27,10 +27,38 @@ #pragma once +#include <string> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/asio/ip/address_v6.hpp> + namespace epee { namespace net_utils { + + inline + bool is_ipv6_local(const std::string& ip) + { + auto addr = boost::asio::ip::make_address_v6(ip); + + // ipv6 link-local unicast addresses are fe80::/10 + bool is_link_local = addr.is_link_local(); + + auto addr_bytes = addr.to_bytes(); + + // ipv6 unique local unicast addresses start with fc00::/7 -- (fcXX or fdXX) + bool is_unique_local_unicast = (addr_bytes[0] == 0xfc || addr_bytes[0] == 0xfd); + + return is_link_local || is_unique_local_unicast; + } + + inline + bool is_ipv6_loopback(const std::string& ip) + { + // ipv6 loopback is ::1 + return boost::asio::ip::address_v6::from_string(ip).is_loopback(); + } + inline bool is_ip_local(uint32_t ip) { diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index 708cce0ff..1d156d19c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -94,7 +94,7 @@ namespace net_utils return true; } - inline + inline bool parse_uri(const std::string uri, http::uri_content& content) { @@ -128,11 +128,51 @@ namespace net_utils return true; } + inline + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 - inline + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + inline bool parse_url(const std::string url_str, http::url_content& content) { + if (parse_url_ipv6(url_str, content)) return true; + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index fce01311c..5ae3e53b3 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -31,6 +31,7 @@ #include <boost/uuid/uuid.hpp> #include <boost/asio/io_service.hpp> +#include <boost/asio/ip/address_v6.hpp> #include <typeinfo> #include <type_traits> #include "enums.h" @@ -154,6 +155,59 @@ namespace net_utils inline bool operator>=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept { return !lhs.less(rhs); } + class ipv6_network_address + { + protected: + boost::asio::ip::address_v6 m_address; + uint16_t m_port; + + public: + ipv6_network_address() + : ipv6_network_address(boost::asio::ip::address_v6::loopback(), 0) + {} + + ipv6_network_address(const boost::asio::ip::address_v6& ip, uint16_t port) + : m_address(ip), m_port(port) + { + } + + bool equal(const ipv6_network_address& other) const noexcept; + bool less(const ipv6_network_address& other) const noexcept; + bool is_same_host(const ipv6_network_address& other) const noexcept + { return m_address == other.m_address; } + + boost::asio::ip::address_v6 ip() const noexcept { return m_address; } + uint16_t port() const noexcept { return m_port; } + std::string str() const; + std::string host_str() const; + bool is_loopback() const; + bool is_local() const; + static constexpr address_type get_type_id() noexcept { return address_type::ipv6; } + static constexpr zone get_zone() noexcept { return zone::public_; } + static constexpr bool is_blockable() noexcept { return true; } + + static const uint8_t ID = 2; + BEGIN_KV_SERIALIZE_MAP() + boost::asio::ip::address_v6::bytes_type bytes = this_ref.m_address.to_bytes(); + epee::serialization::selector<is_store>::serialize_t_val_as_blob(bytes, stg, hparent_section, "addr"); + const_cast<boost::asio::ip::address_v6&>(this_ref.m_address) = boost::asio::ip::address_v6(bytes); + KV_SERIALIZE(m_port) + END_KV_SERIALIZE_MAP() + }; + + inline bool operator==(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return lhs.equal(rhs); } + inline bool operator!=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !lhs.equal(rhs); } + inline bool operator<(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return lhs.less(rhs); } + inline bool operator<=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !rhs.less(lhs); } + inline bool operator>(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return rhs.less(lhs); } + inline bool operator>=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !lhs.less(rhs); } + class network_address { struct interface @@ -261,6 +315,8 @@ namespace net_utils { case address_type::ipv4: return this_ref.template serialize_addr<ipv4_network_address>(is_store_, stg, hparent_section); + case address_type::ipv6: + return this_ref.template serialize_addr<ipv6_network_address>(is_store_, stg, hparent_section); case address_type::tor: return this_ref.template serialize_addr<net::tor_address>(is_store_, stg, hparent_section); case address_type::i2p: diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index b5c4138c5..fe53628a5 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -31,6 +31,9 @@ #include <algorithm> #include <boost/utility/string_ref.hpp> +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + namespace epee { namespace misc_utils @@ -62,6 +65,26 @@ namespace misc_utils 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; + static const constexpr unsigned char isx[256] = + { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; + inline bool isspace(char c) { return lut[(uint8_t)c] & 8; @@ -162,6 +185,42 @@ namespace misc_utils val.push_back('\\');break; case '/': //Slash character val.push_back('/');break; + case 'u': //Unicode code point + if (buf_end - it < 4) + { + ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); + } + else + { + uint32_t dst = 0; + for (int i = 0; i < 4; ++i) + { + const unsigned char tmp = isx[(int)*++it]; + CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); + dst = dst << 4 | tmp; + } + // encode as UTF-8 + if (dst <= 0x7f) + { + val.push_back(dst); + } + else if (dst <= 0x7ff) + { + val.push_back(0xc0 | (dst >> 6)); + val.push_back(0x80 | (dst & 0x3f)); + } + else if (dst <= 0xffff) + { + val.push_back(0xe0 | (dst >> 12)); + val.push_back(0x80 | ((dst >> 6) & 0x3f)); + val.push_back(0x80 | (dst & 0x3f)); + } + else + { + ASSERT_MES_AND_THROW("Unicode code point is out or range"); + } + } + break; default: val.push_back(*it); LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index da47b7d55..1be5eb5e1 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -59,26 +59,6 @@ #pragma comment (lib, "Rpcrt4.lib") #endif -static const constexpr unsigned char isx[256] = -{ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -}; - namespace epee { namespace string_tools @@ -99,10 +79,10 @@ namespace string_tools for(size_t i = 0; i < s.size(); i += 2) { int tmp = *src++; - tmp = isx[tmp]; + tmp = epee::misc_utils::parse::isx[tmp]; if (tmp == 0xff) return false; int t2 = *src++; - t2 = isx[t2]; + t2 = epee::misc_utils::parse::isx[t2]; if (t2 == 0xff) return false; *dst++ = (tmp << 4) | t2; } diff --git a/contrib/epee/src/net_helper.cpp b/contrib/epee/src/net_helper.cpp index 3543f5716..719f1c8e0 100644 --- a/contrib/epee/src/net_helper.cpp +++ b/contrib/epee/src/net_helper.cpp @@ -11,10 +11,39 @@ namespace net_utils ////////////////////////////////////////////////////////////////////////// boost::asio::ip::tcp::resolver resolver(GET_IO_SERVICE(timeout)); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver::iterator iterator; boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty - throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + boost::system::error_code resolve_error; + try + { + iterator = resolver.resolve(query, resolve_error); + if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty + { + // if IPv4 resolution fails, try IPv6. Unintentional outgoing IPv6 connections should only + // be possible if for some reason a hostname was given and that hostname fails IPv4 resolution, + // so at least for now there should not be a need for a flag "using ipv6 is ok" + try_ipv6 = true; + } + + } + catch (const boost::system::system_error& e) + { + if (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again) + { + throw; + } + try_ipv6 = true; + } + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + iterator = resolver.resolve(query6); + if (iterator == end) + throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + } ////////////////////////////////////////////////////////////////////////// diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index b7f07a23b..5cc49cc71 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -21,6 +21,19 @@ namespace epee { namespace net_utils bool ipv4_network_address::is_loopback() const { return net_utils::is_ip_loopback(ip()); } bool ipv4_network_address::is_local() const { return net_utils::is_ip_local(ip()); } + bool ipv6_network_address::equal(const ipv6_network_address& other) const noexcept + { return is_same_host(other) && port() == other.port(); } + + bool ipv6_network_address::less(const ipv6_network_address& other) const noexcept + { return is_same_host(other) ? port() < other.port() : m_address < other.m_address; } + + std::string ipv6_network_address::str() const + { return std::string("[") + host_str() + "]:" + std::to_string(port()); } + + std::string ipv6_network_address::host_str() const { return m_address.to_string(); } + bool ipv6_network_address::is_loopback() const { return m_address.is_loopback(); } + bool ipv6_network_address::is_local() const { return m_address.is_link_local(); } + bool ipv4_network_subnet::equal(const ipv4_network_subnet& other) const noexcept { return is_same_host(other) && m_mask == other.m_mask; } diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index 3fc26354c..32aee5f56 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -26,12 +26,16 @@ Preparing the Gitian builder host The first step is to prepare the host environment that will be used to perform the Gitian builds. This guide explains how to set up the environment, and how to start the builds. -Gitian offers to build with either `kvm`, `docker` or `lxc`. The default build -path chosen is `lxc`, but its setup is more complicated. You need to be logged in as the `gitianuser`. -If this user does not exist yet on your system, create it. Gitian can use -either kvm, lxc or docker as a host environment. This documentation will show -how to build with lxc and docker. While the docker setup is easy, the lxc setup -is more involved. + +* Gitian host OS should be Ubuntu 18.04 "Bionic Beaver". If you are on a mac or windows for example, you can run it in a VM but will be slower. + +* Gitian gives you the option of using any of 3 different virtualization tools: `kvm`, `docker` or `lxc`. This documentation will only show how to build with `lxc` and `docker` (documentation for `kvm` is welcome). Building with `lxc` is the default, but is more complicated, so we recommend docker your first time. + + +## Create the gitianuser account + +You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. + LXC --- @@ -76,10 +80,17 @@ This setup is required to enable networking in the container. Docker ------ -Building in docker does not require much setup. Install docker on your host, then type the following: +Prepare for building with docker: ```bash -sudo apt-get install git make curl +sudo apt-get install git make curl docker.io +``` + +Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): + +```bash +sudo groupadd docker +sudo usermod -aG docker gitianuser ``` Optionally add yourself to the docker group. Note that this will give docker root access to your system. @@ -88,7 +99,7 @@ Optionally add yourself to the docker group. Note that this will give docker roo sudo usermod -aG docker gitianuser ``` -Manual and Building +Manual Building ------------------- The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. @@ -109,66 +120,95 @@ The `gitian-build.py` script will checkout different release tags, so it's best cp monero/contrib/gitian/gitian-build.py . ``` -Setup the required environment, you only need to do this once: +### Setup the required environment + +Setup for LXC: ```bash -./gitian-build.py --setup fluffypony v0.14.0 +GH_USER=fluffypony +VERSION=v0.14.1.0 + +./gitian-build.py --setup $GH_USER $VERSION ``` -Where `fluffypony` is your Github name and `v0.14.0` is the version tag you want to build. -If you are using docker, run it with: +Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. + +Setup for docker: ```bash -./gitian-build.py --setup --docker fluffypony v0.14.0 +./gitian-build.py --setup --docker $GH_USER $VERSION ``` -While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. -This script is only there for convenience. Seperate steps for building can still be taken. +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a separate step. This script is only there for convenience. Separate steps for building can still be taken. In order to sign gitian builds on your host machine, which has your PGP key, -fork the gitian.sigs repository and clone it on your host machine, +fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) and clone it on your host machine, or pass the signed assert file back to your build machine. ```bash git clone git@github.com:monero-project/gitian.sigs.git -git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git +git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git ``` -Build Binaries ------------------------------ -To build the most recent tag (pass in `--docker` after setting up with docker): +Build the binaries +------------------ + +**Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. + +To build the most recent tag (pass in `--docker` if using docker): ```bash -./gitian-build.py --detach-sign --no-commit -b fluffypony v0.14.0 +./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION ``` -To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. +To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. -If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. +If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. -If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: +Checking your work +------------------ + +Take a look in the assert files and note the SHA256 checksums listed there. eg for `v0.14.1.0` you should get this checksum: + +``` +2b95118f53d98d542a85f8732b84ba13b3cd20517ccb40332b0edd0ddf4f8c62 monero-x86_64-linux-gnu.tar.gz +``` + +You should verify that this is really the checksum you get on that file you built. You can also look in the gitian.sigs repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. + + +Signing assert files +-------------------- + +If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -export NAME=fluffypony -export VERSION=v0.14.0 -gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert -gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert -gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert +GH_USER=fluffypony +VERSION=v0.14.1.0 + +gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert +gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert +gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert ``` +<!-- TODO: Replace * above with ${VERSION} once gitian builds correct file name --> + +This will create a `.sig` file for each `.assert` file above (2 files for each platform). + + +Submitting your signed assert files +----------------------------------- Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash -git checkout -b v0.14.0 -git commit -S -a -m "Add $NAME v0.14.0" -git push --set-upstream $NAME v0.14.0 +git checkout -b $VERSION +# add your assert and sig files... +git commit -S -a -m "Add $GH_USER $VERSION" +git push --set-upstream $GH_USER $VERSION ``` -```bash -gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert -gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert -gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert -``` +**Note:** Please ensure your gpg public key is available to check signatures by adding it to the [gitian.sigs/gitian-pubkeys/](https://github.com/monero-project/gitian.sigs/tree/master/gitian-pubkeys) directory in a pull request. + More Build Options ------------------ diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 6e3ad108e..fd94d43bf 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -131,6 +131,13 @@ script: | git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host for i in $HOSTS; do @@ -155,9 +162,10 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON make ${MAKEOPTS} - DISTNAME=monero-${i} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz cd .. rm -rf build done + diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 1cdb85273..77ea30072 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -79,6 +79,13 @@ script: | git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends mkdir -p ${BASEPREFIX}/SDKs @@ -102,7 +109,7 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make ${MAKEOPTS} - DISTNAME=monero-${i} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz cd .. diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index d15383905..4c559acfe 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -102,6 +102,13 @@ script: | git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host for i in $HOSTS; do @@ -127,7 +134,7 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make ${MAKEOPTS} - DISTNAME=monero-${i} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip cd .. && rm -rf build diff --git a/external/boost/archive/portable_binary_archive.hpp b/external/boost/archive/portable_binary_archive.hpp index e940c5b9e..7ae01a225 100644 --- a/external/boost/archive/portable_binary_archive.hpp +++ b/external/boost/archive/portable_binary_archive.hpp @@ -22,7 +22,7 @@ #endif
#include <boost/archive/basic_archive.hpp>
-#include <boost/detail/endian.hpp>
+#include <boost/predef/other/endian.h>
#include <boost/archive/impl/archive_serializer_map.ipp>
diff --git a/external/boost/archive/portable_binary_iarchive.hpp b/external/boost/archive/portable_binary_iarchive.hpp index 7792b530d..bd19599f3 100644 --- a/external/boost/archive/portable_binary_iarchive.hpp +++ b/external/boost/archive/portable_binary_iarchive.hpp @@ -226,7 +226,7 @@ public: #include <istream>
#include <string>
-#include <boost/detail/endian.hpp>
+#include <boost/predef/other/endian.h>
#include <boost/serialization/throw_exception.hpp>
#include <boost/archive/archive_exception.hpp>
@@ -252,12 +252,12 @@ portable_binary_iarchive::load_impl(boost::intmax_t & l, char maxsize){ );
char * cptr = reinterpret_cast<char *>(& l);
- #ifdef BOOST_BIG_ENDIAN
+ #if BOOST_ENDIAN_BIG_BYTE
cptr += (sizeof(boost::intmax_t) - size);
#endif
this->primitive_base_t::load_binary(cptr, size);
- #ifdef BOOST_BIG_ENDIAN
+ #if BOOST_ENDIAN_BIG_BYTE
if(m_flags & endian_little)
#else
if(m_flags & endian_big)
diff --git a/external/boost/archive/portable_binary_oarchive.hpp b/external/boost/archive/portable_binary_oarchive.hpp index e2dcb9456..783c7f7c9 100644 --- a/external/boost/archive/portable_binary_oarchive.hpp +++ b/external/boost/archive/portable_binary_oarchive.hpp @@ -221,7 +221,7 @@ public: // See http://www.boost.org for updates, documentation, and revision history.
#include <ostream>
-#include <boost/detail/endian.hpp>
+#include <boost/predef/other/endian.h>
namespace boost { namespace archive {
@@ -258,7 +258,7 @@ portable_binary_oarchive::save_impl( else
ll = l;
char * cptr = reinterpret_cast<char *>(& ll);
- #ifdef BOOST_BIG_ENDIAN
+ #if BOOST_ENDIAN_BIG_BYTE
cptr += (sizeof(boost::intmax_t) - size);
if(m_flags & endian_little)
reverse_bytes(size, cptr);
diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 5a806dc69..2b4c7bbbf 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -1969,7 +1969,7 @@ void RegisteredLoggers::unsafeFlushAll(void) { // VRegistry -VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags) { +VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags), m_lowest_priority(INT_MAX) { } /// @brief Sets verbose level. Accepted range is 0-9 @@ -2053,14 +2053,30 @@ void VRegistry::setModules(const char* modules) { } } +// Log levels are sorted in a weird way... +static int priority(Level level) { + if (level == Level::Fatal) return 0; + if (level == Level::Error) return 1; + if (level == Level::Warning) return 2; + if (level == Level::Info) return 3; + if (level == Level::Debug) return 4; + if (level == Level::Verbose) return 5; + if (level == Level::Trace) return 6; + return 7; +} + void VRegistry::setCategories(const char* categories, bool clear) { base::threading::ScopedLock scopedLock(lock()); auto insert = [&](std::stringstream& ss, Level level) { m_categories.push_back(std::make_pair(ss.str(), level)); m_cached_allowed_categories.clear(); + int pri = priority(level); + if (pri > m_lowest_priority) + m_lowest_priority = pri; }; if (clear) { + m_lowest_priority = 0; m_categories.clear(); m_cached_allowed_categories.clear(); m_categoriesString.clear(); @@ -2117,23 +2133,14 @@ std::string VRegistry::getCategories() { return m_categoriesString; } -// Log levels are sorted in a weird way... -static int priority(Level level) { - if (level == Level::Fatal) return 0; - if (level == Level::Error) return 1; - if (level == Level::Warning) return 2; - if (level == Level::Info) return 3; - if (level == Level::Debug) return 4; - if (level == Level::Verbose) return 5; - if (level == Level::Trace) return 6; - return 7; -} - bool VRegistry::allowed(Level level, const std::string &category) { + const int pri = priority(level); + if (pri > m_lowest_priority) + return false; base::threading::ScopedLock scopedLock(lock()); const std::map<std::string, int>::const_iterator it = m_cached_allowed_categories.find(category); if (it != m_cached_allowed_categories.end()) - return priority(level) <= it->second; + return pri <= it->second; if (m_categories.empty()) { return false; } else { @@ -2142,7 +2149,7 @@ bool VRegistry::allowed(Level level, const std::string &category) { if (base::utils::Str::wildCardMatch(category.c_str(), it->first.c_str())) { const int p = priority(it->second); m_cached_allowed_categories.insert(std::make_pair(category, p)); - return priority(level) <= p; + return pri <= p; } } m_cached_allowed_categories.insert(std::make_pair(category, -1)); diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index f0d8d5df7..f1fa2cb0d 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -359,6 +359,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #if defined(ELPP_SYSLOG) # include <syslog.h> #endif // defined(ELPP_SYSLOG) +#include <climits> #include <ctime> #include <cstring> #include <cstdlib> @@ -406,6 +407,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #include <sstream> #include <memory> #include <type_traits> +#include <atomic> #if ELPP_THREADING_ENABLED # if ELPP_USE_STD_THREADING # include <mutex> @@ -2451,6 +2453,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { base::threading::ScopedLock scopedLock(lock()); m_categories.clear(); m_cached_allowed_categories.clear(); + m_lowest_priority = INT_MAX; } inline void clearModules(void) { @@ -2495,6 +2498,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { std::map<std::string, int> m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; + std::atomic<int> m_lowest_priority; }; } // namespace base class LogMessage { diff --git a/src/common/util.cpp b/src/common/util.cpp index db5aa3052..0fa9e8dc1 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1068,6 +1068,23 @@ std::string get_nix_version_display_string() return std::string(buffer); } + std::string get_human_readable_timespan(uint64_t seconds) + { + if (seconds < 60) + return std::to_string(seconds) + " seconds"; + if (seconds < 3600) + return std::to_string((uint64_t)(seconds / 60)) + " minutes"; + if (seconds < 3600 * 24) + return std::to_string((uint64_t)(seconds / 3600)) + " hours"; + if (seconds < 3600 * 24 * 30.5) + return std::to_string((uint64_t)(seconds / (3600 * 24))) + " days"; + if (seconds < 3600 * 24 * 365.25) + return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5))) + " months"; + if (seconds < 3600 * 24 * 365.25 * 100) + return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5 * 365.25))) + " years"; + return "a long time"; + } + std::string get_human_readable_bytes(uint64_t bytes) { // Use 1024 for "kilo", 1024*1024 for "mega" and so on instead of the more modern and standard-conforming diff --git a/src/common/util.h b/src/common/util.h index f6d5c9b1f..b0f734eff 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -245,5 +245,7 @@ namespace tools std::string get_human_readable_timestamp(uint64_t ts); + std::string get_human_readable_timespan(uint64_t seconds); + std::string get_human_readable_bytes(uint64_t bytes); } diff --git a/src/crypto/CryptonightR_JIT.c b/src/crypto/CryptonightR_JIT.c index ee8f3f36f..ffe7820a2 100644 --- a/src/crypto/CryptonightR_JIT.c +++ b/src/crypto/CryptonightR_JIT.c @@ -63,7 +63,7 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f #if !(defined(_MSC_VER) || defined(__MINGW32__)) if (mprotect((void*)buf, buf_size, PROT_READ | PROT_WRITE)) - return 1; + return -1; #endif APPEND_CODE(prologue, sizeof(prologue)); @@ -111,13 +111,13 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f #if !(defined(_MSC_VER) || defined(__MINGW32__)) if (mprotect((void*)buf, buf_size, PROT_READ | PROT_EXEC)) - return 1; + return -1; #endif __builtin___clear_cache((char*)buf, (char*)JIT_code); return 0; #else - return 1; + return -1; #endif } diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index e594eb049..2dad2795e 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -91,7 +91,7 @@ namespace cryptonote const command_line::arg_descriptor<std::string> arg_extra_messages = {"extra-messages-file", "Specify file for extra messages to include into coinbase transactions", "", true}; const command_line::arg_descriptor<std::string> arg_start_mining = {"start-mining", "Specify wallet address to mining for", "", true}; const command_line::arg_descriptor<uint32_t> arg_mining_threads = {"mining-threads", "Specify mining threads count", 0, true}; - const command_line::arg_descriptor<bool> arg_bg_mining_enable = {"bg-mining-enable", "enable/disable background mining", true, true}; + const command_line::arg_descriptor<bool> arg_bg_mining_enable = {"bg-mining-enable", "enable background mining", true, true}; const command_line::arg_descriptor<bool> arg_bg_mining_ignore_battery = {"bg-mining-ignore-battery", "if true, assumes plugged in when unable to query system power status", false, true}; const command_line::arg_descriptor<uint64_t> arg_bg_mining_min_idle_interval_seconds = {"bg-mining-min-idle-interval", "Specify min lookback interval in seconds for determining idle state", miner::BACKGROUND_MINING_DEFAULT_MIN_IDLE_INTERVAL_IN_SECONDS, true}; const command_line::arg_descriptor<uint16_t> arg_bg_mining_idle_threshold_percentage = {"bg-mining-idle-threshold", "Specify minimum avg idle percentage over lookback interval", miner::BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE, true}; diff --git a/src/cryptonote_basic/verification_context.h b/src/cryptonote_basic/verification_context.h index 36b63f254..3d7200fae 100644 --- a/src/cryptonote_basic/verification_context.h +++ b/src/cryptonote_basic/verification_context.h @@ -48,6 +48,7 @@ namespace cryptonote bool m_overspend; bool m_fee_too_low; bool m_not_rct; + bool m_too_few_outputs; }; struct block_verification_context diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 56b6a63b7..b68bb41e1 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -128,6 +128,8 @@ #define P2P_SUPPORT_FLAG_FLUFFY_BLOCKS 0x01 #define P2P_SUPPORT_FLAGS P2P_SUPPORT_FLAG_FLUFFY_BLOCKS +#define RPC_IP_FAILS_BEFORE_BLOCK 3 + #define ALLOW_DEBUG_COMMANDS #define CRYPTONOTE_NAME "bitmonero" @@ -147,6 +149,7 @@ #define HF_VERSION_PER_BYTE_FEE 8 #define HF_VERSION_SMALLER_BP 10 #define HF_VERSION_LONG_TERM_BLOCK_WEIGHT 10 +#define HF_VERSION_MIN_2_OUTPUTS 12 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 31e2ac136..dab1a0a96 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -715,7 +715,6 @@ block Blockchain::pop_block_from_blockchain() m_blocks_longhash_table.clear(); m_scan_table.clear(); m_blocks_txs_check.clear(); - m_check_txin_table.clear(); CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); uint64_t top_block_height; @@ -2612,7 +2611,7 @@ void Blockchain::on_new_tx_from_block(const cryptonote::transaction &tx) // This function overloads its sister function with // an extra value (hash of highest block that holds an output used as input) // as a return-by-reference. -bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block) +bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2643,7 +2642,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh return true; } //------------------------------------------------------------------ -bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc) +bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2752,7 +2751,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const } return false; } -bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) +bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const { PERF_TIMER(expand_transaction_2); CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2"); @@ -2828,7 +2827,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr // check_tx_input() rather than here, and use this function simply // to iterate the inputs as necessary (splitting the task // using threads, etc.) -bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height) +bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height) const { PERF_TIMER(check_tx_inputs); LOG_PRINT_L3("Blockchain::" << __func__); @@ -2840,6 +2839,19 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, const uint8_t hf_version = m_hardfork->get_current_version(); + if (hf_version >= HF_VERSION_MIN_2_OUTPUTS) + { + if (tx.version >= 2) + { + if (tx.vout.size() < 2) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has fewer than two outputs"); + tvc.m_too_few_outputs = true; + return false; + } + } + } + // from hard fork 2, we require mixin at least 2 unless one output cannot mix with 2 others // if one output cannot mix with 2 others, we accept at most 1 output that can mix if (hf_version >= 2) @@ -2934,13 +2946,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } } - auto it = m_check_txin_table.find(tx_prefix_hash); - if(it == m_check_txin_table.end()) - { - m_check_txin_table.emplace(tx_prefix_hash, std::unordered_map<crypto::key_image, bool>()); - it = m_check_txin_table.find(tx_prefix_hash); - assert(it != m_check_txin_table.end()); - } std::vector<std::vector<rct::ctkey>> pubkeys(tx.vin.size()); std::vector < uint64_t > results; @@ -2972,29 +2977,12 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { // basically, make sure number of inputs == number of signatures CHECK_AND_ASSERT_MES(sig_index < tx.signatures.size(), false, "wrong transaction: not signature entry for input with index= " << sig_index); - -#if defined(CACHE_VIN_RESULTS) - auto itk = it->second.find(in_to_key.k_image); - if(itk != it->second.end()) - { - if(!itk->second) - { - MERROR_VER("Failed ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); - return false; - } - - // txin has been verified already, skip - sig_index++; - continue; - } -#endif } // make sure that output being spent matches up correctly with the // signature spending it. if (!check_tx_input(tx.version, in_to_key, tx_prefix_hash, tx.version == 1 ? tx.signatures[sig_index] : std::vector<crypto::signature>(), tx.rct_signatures, pubkeys[sig_index], pmax_used_block_height)) { - it->second[in_to_key.k_image] = false; MERROR_VER("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain() { @@ -3017,7 +3005,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, check_ring_signature(tx_prefix_hash, in_to_key.k_image, pubkeys[sig_index], tx.signatures[sig_index], results[sig_index]); if (!results[sig_index]) { - it->second[in_to_key.k_image] = false; MERROR_VER("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain() @@ -3027,7 +3014,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, return false; } - it->second[in_to_key.k_image] = true; } } @@ -3045,7 +3031,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, for (size_t i = 0; i < tx.vin.size(); i++) { const txin_to_key& in_to_key = boost::get<txin_to_key>(tx.vin[i]); - it->second[in_to_key.k_image] = results[i]; if(!failed && !results[i]) failed = true; } @@ -3219,7 +3204,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } //------------------------------------------------------------------ -void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature>& sig, uint64_t &result) +void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature>& sig, uint64_t &result) const { std::vector<const crypto::public_key *> p_output_keys; p_output_keys.reserve(pubkeys.size()); @@ -3405,7 +3390,7 @@ bool Blockchain::is_tx_spendtime_unlocked(uint64_t unlock_time) const // This function locates all outputs associated with a given input (mixins) // and validates that they exist and are usable. It also checks the ring // signature for each input. -bool Blockchain::check_tx_input(size_t tx_version, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, const rct::rctSig &rct_signatures, std::vector<rct::ctkey> &output_keys, uint64_t* pmax_related_block_height) +bool Blockchain::check_tx_input(size_t tx_version, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, const rct::rctSig &rct_signatures, std::vector<rct::ctkey> &output_keys, uint64_t* pmax_related_block_height) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -4242,7 +4227,6 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) m_blocks_longhash_table.clear(); m_scan_table.clear(); m_blocks_txs_check.clear(); - m_check_txin_table.clear(); // when we're well clear of the precomputed hashes, free the memory if (!m_blocks_hash_check.empty() && m_db->height() > m_blocks_hash_check.size() + 4096) @@ -4531,7 +4515,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete m_fake_pow_calc_time = 0; m_scan_table.clear(); - m_check_txin_table.clear(); TIME_MEASURE_FINISH(prepare); m_fake_pow_calc_time = prepare / blocks_entry.size(); @@ -4840,9 +4823,9 @@ std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> Blockchain:: get_ou return m_db->get_output_histogram(amounts, unlocked, recent_cutoff, min_count); } -std::list<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> Blockchain::get_alternative_chains() const +std::vector<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> Blockchain::get_alternative_chains() const { - std::list<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> chains; + std::vector<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> chains; blocks_ext_by_hash alt_blocks; alt_blocks.reserve(m_db->get_alt_block_count()); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index b1c23b704..f58059a6d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -548,7 +548,7 @@ namespace cryptonote * * @return false if any input is invalid, otherwise true */ - bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); + bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false) const; /** * @brief get fee quantization mask @@ -615,7 +615,7 @@ namespace cryptonote * * @return false if any outputs do not conform, otherwise true */ - bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); + bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc) const; /** * @brief gets the block weight limit based on recent blocks @@ -950,9 +950,9 @@ namespace cryptonote /** * @brief returns a set of known alternate chains * - * @return a list of chains + * @return a vector of chains */ - std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const; + std::vector<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const; void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); @@ -1020,7 +1020,6 @@ namespace cryptonote // metadata containers std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table; - std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, bool>> m_check_txin_table; // SHA-3 hashes for each block and for fast pow checking std::vector<crypto::hash> m_blocks_hash_of_hashes; @@ -1127,7 +1126,7 @@ namespace cryptonote * * @return false if any output is not yet unlocked, or is missing, otherwise true */ - bool check_tx_input(size_t tx_version,const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, const rct::rctSig &rct_signatures, std::vector<rct::ctkey> &output_keys, uint64_t* pmax_related_block_height); + bool check_tx_input(size_t tx_version,const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, const rct::rctSig &rct_signatures, std::vector<rct::ctkey> &output_keys, uint64_t* pmax_related_block_height) const; /** * @brief validate a transaction's inputs and their keys @@ -1149,7 +1148,7 @@ namespace cryptonote * * @return false if any validation step fails, otherwise true */ - bool check_tx_inputs(transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height = NULL); + bool check_tx_inputs(transaction& tx, tx_verification_context &tvc, uint64_t* pmax_used_block_height = NULL) const; /** * @brief performs a blockchain reorganization according to the longest chain rule @@ -1429,7 +1428,7 @@ namespace cryptonote * @param result false if the ring signature is invalid, otherwise true */ void check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, - const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature> &sig, uint64_t &result); + const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature> &sig, uint64_t &result) const; /** * @brief loads block hashes from compiled-in data set @@ -1449,7 +1448,7 @@ namespace cryptonote * can be reconstituted by the receiver. This function expands * that implicit data. */ - bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys); + bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const; /** * @brief invalidates any cached block template diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 2ebd99904..a3a92ab60 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1844,7 +1844,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::check_block_rate() { - if (m_offline || m_nettype == FAKECHAIN || m_target_blockchain_height > get_current_blockchain_height()) + if (m_offline || m_nettype == FAKECHAIN || m_target_blockchain_height > get_current_blockchain_height() || m_target_blockchain_height == 0) { MDEBUG("Not checking block rate, offline or syncing"); return true; diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 854f3d1c5..4cf71e558 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -354,7 +354,7 @@ namespace cryptonote if (shuffle_outs) { - std::shuffle(destinations.begin(), destinations.end(), std::default_random_engine(crypto::rand<unsigned int>())); + std::shuffle(destinations.begin(), destinations.end(), crypto::random_device{}); } // sort ins by their key image diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index d38aa7474..b03eb6e70 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -83,6 +83,21 @@ namespace cryptonote tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } tx_destination_entry(const std::string &o, uint64_t a, const account_public_address &ad, bool is_subaddress) : original(o), amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } + std::string address(network_type nettype, const crypto::hash &payment_id) const + { + if (!original.empty()) + { + return original; + } + + if (is_integrated) + { + return get_account_integrated_address_as_str(nettype, addr, reinterpret_cast<const crypto::hash8 &>(payment_id)); + } + + return get_account_address_as_str(nettype, is_subaddress, addr); + } + BEGIN_SERIALIZE_OBJECT() FIELD(original) VARINT_FIELD(amount) diff --git a/src/cryptonote_core/tx_sanity_check.cpp b/src/cryptonote_core/tx_sanity_check.cpp index e95350f76..03cbb5c26 100644 --- a/src/cryptonote_core/tx_sanity_check.cpp +++ b/src/cryptonote_core/tx_sanity_check.cpp @@ -82,7 +82,7 @@ bool tx_sanity_check(Blockchain &blockchain, const cryptonote::blobdata &tx_blob if (rct_indices.size() < n_indices * 8 / 10) { - MERROR("unique indices is only " << rct_indices.size() << "/" << n_indices); + MERROR("amount of unique indices is too low (amount of rct indices is " << rct_indices.size() << ", out of total " << n_indices << "indices."); return false; } @@ -90,7 +90,7 @@ bool tx_sanity_check(Blockchain &blockchain, const cryptonote::blobdata &tx_blob uint64_t median = epee::misc_utils::median(offsets); if (median < n_available * 6 / 10) { - MERROR("median is " << median << "/" << n_available); + MERROR("median offset index is too low (median is " << median << " out of total " << n_available << "offsets). Transactions should contain a higher fraction of recent outputs."); return false; } diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 778d7b4d8..924447701 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -674,11 +674,38 @@ bool t_command_parser_executor::alt_chain_info(const std::vector<std::string>& a { if(args.size() > 1) { - std::cout << "usage: alt_chain_info [block_hash]" << std::endl; + std::cout << "usage: alt_chain_info [block_hash|>N|-N]" << std::endl; return false; } - return m_executor.alt_chain_info(args.size() == 1 ? args[0] : ""); + std::string tip; + size_t above = 0; + uint64_t last_blocks = 0; + if (args.size() == 1) + { + if (args[0].size() > 0 && args[0][0] == '>') + { + if (!epee::string_tools::get_xtype_from_string(above, args[0].c_str() + 1)) + { + std::cout << "invalid above parameter" << std::endl; + return false; + } + } + else if (args[0].size() > 0 && args[0][0] == '-') + { + if (!epee::string_tools::get_xtype_from_string(last_blocks, args[0].c_str() + 1)) + { + std::cout << "invalid last_blocks parameter" << std::endl; + return false; + } + } + else + { + tip = args[0]; + } + } + + return m_executor.alt_chain_info(tip, above, last_blocks); } bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector<std::string>& args) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index ea986a398..4d3debed6 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -35,6 +35,7 @@ #include "daemon/rpc_command_executor.h" #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_basic/difficulty.h" #include "cryptonote_basic/hardfork.h" #include <boost/format.hpp> #include <ctime> @@ -65,15 +66,11 @@ namespace { time(&now); time_t last_seen = static_cast<time_t>(peer.last_seen); - std::string id_str; + std::string elapsed = epee::misc_utils::get_time_interval_string(now - last_seen); + std::string id_str = epee::string_tools::pad_string(epee::string_tools::to_string_hex(peer.id), 16, '0', true); std::string port_str; - std::string elapsed = peer.last_seen == 0 ? "never" : epee::misc_utils::get_time_interval_string(now - last_seen); - std::string ip_str = epee::string_tools::get_ip_string_from_int32(peer.ip); - std::stringstream peer_id_str; - peer_id_str << std::hex << std::setw(16) << peer.id; - peer_id_str >> id_str; epee::string_tools::xtype_to_string(peer.port, port_str); - std::string addr_str = ip_str + ":" + port_str; + std::string addr_str = peer.host + ":" + port_str; std::string rpc_port = peer.rpc_port ? std::to_string(peer.rpc_port) : "-"; std::string pruning_seed = epee::string_tools::to_string_hex(peer.pruning_seed); tools::msg_writer() << boost::format("%-10s %-25s %-25s %-5s %-4s %s") % prefix % id_str % addr_str % rpc_port % pruning_seed % elapsed; @@ -89,7 +86,7 @@ namespace { << "height: " << boost::lexical_cast<std::string>(header.height) << std::endl << "depth: " << boost::lexical_cast<std::string>(header.depth) << std::endl << "hash: " << header.hash << std::endl - << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl + << "difficulty: " << header.wide_difficulty << std::endl << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl << "block weight: " << header.block_weight << std::endl @@ -221,6 +218,9 @@ bool t_rpc_command_executor::print_peer_list_stats() { cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; std::string failure_message = "Couldn't retrieve peer list"; + + req.public_only = false; + if (m_is_rpc) { if (!m_rpc_client->rpc_request(req, res, "/get_peer_list", failure_message.c_str())) @@ -350,19 +350,41 @@ bool t_rpc_command_executor::show_difficulty() { tools::success_msg_writer() << "BH: " << res.height << ", TH: " << res.top_block_hash - << ", DIFF: " << res.difficulty - << ", CUM_DIFF: " << res.cumulative_difficulty - << ", HR: " << res.difficulty / res.target << " H/s"; + << ", DIFF: " << res.wide_difficulty + << ", CUM_DIFF: " << res.wide_cumulative_difficulty + << ", HR: " << cryptonote::difficulty_type(res.wide_difficulty) / res.target << " H/s"; return true; } -static std::string get_mining_speed(uint64_t hr) +static void get_metric_prefix(cryptonote::difficulty_type hr, double& hr_d, char& prefix) +{ + if (hr < 1000) + { + prefix = 0; + return; + } + static const char metric_prefixes[4] = { 'k', 'M', 'G', 'T' }; + for (size_t i = 0; i < sizeof(metric_prefixes); ++i) + { + if (hr < 1000000) + { + hr_d = hr.convert_to<double>() / 1000; + prefix = metric_prefixes[i]; + return; + } + hr /= 1000; + } + prefix = 0; +} + +static std::string get_mining_speed(cryptonote::difficulty_type hr) { - if (hr>1e9) return (boost::format("%.2f GH/s") % (hr/1e9)).str(); - if (hr>1e6) return (boost::format("%.2f MH/s") % (hr/1e6)).str(); - if (hr>1e3) return (boost::format("%.2f kH/s") % (hr/1e3)).str(); - return (boost::format("%.0f H/s") % hr).str(); + double hr_d; + char prefix; + get_metric_prefix(hr, hr_d, prefix); + if (prefix == 0) return (boost::format("%.0f H/s") % hr).str(); + return (boost::format("%.2f %cH/s") % hr_d % prefix).str(); } static std::string get_fork_extra_info(uint64_t t, uint64_t now, uint64_t block_time) @@ -479,7 +501,7 @@ bool t_rpc_command_executor::show_status() { % (ires.testnet ? "testnet" : ires.stagenet ? "stagenet" : "mainnet") % bootstrap_msg % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed)) : "not mining") - % get_mining_speed(ires.difficulty / ires.target) + % get_mining_speed(cryptonote::difficulty_type(ires.wide_difficulty) / ires.target) % (unsigned)hfres.version % get_fork_extra_info(hfres.earliest_height, net_height, ires.target) % (hfres.state == cryptonote::HardFork::Ready ? "up to date" : hfres.state == cryptonote::HardFork::UpdateNeeded ? "update needed" : "out of date, likely forked") @@ -742,7 +764,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u << ", size: " << header.block_size << ", weight: " << header.block_weight << " (long term " << header.long_term_weight << "), transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl - << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; + << "difficulty: " << header.wide_difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; first = false; } @@ -1818,7 +1840,7 @@ bool t_rpc_command_executor::print_coinbase_tx_sum(uint64_t height, uint64_t cou return true; } -bool t_rpc_command_executor::alt_chain_info(const std::string &tip) +bool t_rpc_command_executor::alt_chain_info(const std::string &tip, size_t above, uint64_t last_blocks) { cryptonote::COMMAND_RPC_GET_INFO::request ireq; cryptonote::COMMAND_RPC_GET_INFO::response ires; @@ -1855,16 +1877,31 @@ bool t_rpc_command_executor::alt_chain_info(const std::string &tip) if (tip.empty()) { - tools::msg_writer() << boost::lexical_cast<std::string>(res.chains.size()) << " alternate chains found:"; - for (const auto &chain: res.chains) + auto chains = res.chains; + std::sort(chains.begin(), chains.end(), [](const cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info &info0, cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info &info1){ return info0.height < info1.height; }); + std::vector<size_t> display; + for (size_t i = 0; i < chains.size(); ++i) { - uint64_t start_height = (chain.height - chain.length + 1); + const auto &chain = chains[i]; + if (chain.length <= above) + continue; + const uint64_t start_height = (chain.height - chain.length + 1); + if (last_blocks > 0 && ires.height - 1 - start_height >= last_blocks) + continue; + display.push_back(i); + } + tools::msg_writer() << boost::lexical_cast<std::string>(display.size()) << " alternate chains found:"; + for (const size_t idx: display) + { + const auto &chain = chains[idx]; + const uint64_t start_height = (chain.height - chain.length + 1); tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) - << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + << " deep), diff " << chain.wide_difficulty << ": " << chain.block_hash; } } else { + const uint64_t now = time(NULL); const auto i = std::find_if(res.chains.begin(), res.chains.end(), [&tip](cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info &info){ return info.block_hash == tip; }); if (i != res.chains.end()) { @@ -1872,10 +1909,53 @@ bool t_rpc_command_executor::alt_chain_info(const std::string &tip) tools::success_msg_writer() << "Found alternate chain with tip " << tip; uint64_t start_height = (chain.height - chain.length + 1); tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) - << " deep), diff " << chain.difficulty << ":"; + << " deep), diff " << chain.wide_difficulty << ":"; for (const std::string &block_id: chain.block_hashes) tools::msg_writer() << " " << block_id; tools::msg_writer() << "Chain parent on main chain: " << chain.main_chain_parent_block; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request bhreq; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response bhres; + bhreq.hashes = chain.block_hashes; + bhreq.hashes.push_back(chain.main_chain_parent_block); + bhreq.fill_pow_hash = false; + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(bhreq, bhres, "getblockheaderbyhash", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_block_header_by_hash(bhreq, bhres, error_resp)) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + if (bhres.block_headers.size() != chain.length + 1) + { + tools::fail_msg_writer() << "Failed to get block header info for alt chain"; + return true; + } + uint64_t t0 = bhres.block_headers.front().timestamp, t1 = t0; + for (const cryptonote::block_header_response &block_header: bhres.block_headers) + { + t0 = std::min<uint64_t>(t0, block_header.timestamp); + t1 = std::max<uint64_t>(t1, block_header.timestamp); + } + const uint64_t dt = t1 - t0; + const uint64_t age = std::max(dt, t0 < now ? now - t0 : 0); + tools::msg_writer() << "Age: " << tools::get_human_readable_timespan(age); + if (chain.length > 1) + { + tools::msg_writer() << "Time span: " << tools::get_human_readable_timespan(dt); + cryptonote::difficulty_type start_difficulty = bhres.block_headers.back().difficulty; + if (start_difficulty > 0) + tools::msg_writer() << "Approximated " << 100.f * DIFFICULTY_TARGET_V2 * chain.length / dt << "% of network hash rate"; + else + tools::fail_msg_writer() << "Bad cmumulative difficulty reported by dameon"; + } } else tools::fail_msg_writer() << "Block hash " << tip << " is not the tip of any known alternate chain"; @@ -1933,7 +2013,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) } } - tools::msg_writer() << "Height: " << ires.height << ", diff " << ires.difficulty << ", cum. diff " << ires.cumulative_difficulty + tools::msg_writer() << "Height: " << ires.height << ", diff " << ires.wide_difficulty << ", cum. diff " << ires.wide_cumulative_difficulty << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/" << (hfres.enabled ? "byte" : "kB"); if (nblocks > 0) @@ -1960,7 +2040,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) } } - double avgdiff = 0; + cryptonote::difficulty_type avgdiff = 0; double avgnumtxes = 0; double avgreward = 0; std::vector<uint64_t> weights; @@ -1969,7 +2049,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) std::vector<unsigned> major_versions(256, 0), minor_versions(256, 0); for (const auto &bhr: bhres.headers) { - avgdiff += bhr.difficulty; + avgdiff += cryptonote::difficulty_type(bhr.wide_difficulty); avgnumtxes += bhr.num_txes; avgreward += bhr.reward; weights.push_back(bhr.block_weight); @@ -1984,7 +2064,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgnumtxes /= nblocks; avgreward /= nblocks; uint64_t median_block_weight = epee::misc_utils::median(weights); - tools::msg_writer() << "Last " << nblocks << ": avg. diff " << (uint64_t)avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes + tools::msg_writer() << "Last " << nblocks << ": avg. diff " << avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block weight " << median_block_weight; unsigned int max_major = 256, max_minor = 256; diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 4622609ae..f3ed48319 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -145,7 +145,7 @@ public: bool print_coinbase_tx_sum(uint64_t height, uint64_t count); - bool alt_chain_info(const std::string &tip); + bool alt_chain_info(const std::string &tip, size_t above, uint64_t last_blocks); bool print_blockchain_dynamic_stats(uint64_t nblocks); diff --git a/src/net/parse.cpp b/src/net/parse.cpp index d93d7d352..7b74f2b75 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -34,28 +34,69 @@ namespace net { + void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port) + { + // require ipv6 address format "[addr:addr:addr:...:addr]:port" + if (address.find(']') != std::string::npos) + { + host = address.substr(1, address.rfind(']') - 1); + if ((host.size() + 2) < address.size()) + { + port = address.substr(address.rfind(':') + 1); + } + } + else + { + host = address.substr(0, address.rfind(':')); + if (host.size() < address.size()) + { + port = address.substr(host.size() + 1); + } + } + } + expect<epee::net_utils::network_address> get_network_address(const boost::string_ref address, const std::uint16_t default_port) { - const boost::string_ref host = address.substr(0, address.rfind(':')); + std::string host_str = ""; + std::string port_str = ""; + + bool ipv6 = false; + + get_network_address_host_and_port(std::string(address), host_str, port_str); - if (host.empty()) + boost::string_ref host_str_ref(host_str); + boost::string_ref port_str_ref(port_str); + + if (host_str.empty()) return make_error_code(net::error::invalid_host); - if (host.ends_with(".onion")) + if (host_str_ref.ends_with(".onion")) return tor_address::make(address, default_port); - if (host.ends_with(".i2p")) + if (host_str_ref.ends_with(".i2p")) return i2p_address::make(address, default_port); + boost::system::error_code ec; + boost::asio::ip::address_v6 v6 = boost::asio::ip::make_address_v6(host_str, ec); + ipv6 = !ec; + std::uint16_t port = default_port; - if (host.size() < address.size()) + if (port_str.size()) { - if (!epee::string_tools::get_xtype_from_string(port, std::string{address.substr(host.size() + 1)})) + if (!epee::string_tools::get_xtype_from_string(port, port_str)) return make_error_code(net::error::invalid_port); } - std::uint32_t ip = 0; - if (epee::string_tools::get_ip_int32_from_string(ip, std::string{host})) - return {epee::net_utils::ipv4_network_address{ip, port}}; + if (ipv6) + { + return {epee::net_utils::ipv6_network_address{v6, port}}; + } + else + { + std::uint32_t ip = 0; + if (epee::string_tools::get_ip_int32_from_string(ip, host_str)) + return {epee::net_utils::ipv4_network_address{ip, port}}; + } + return make_error_code(net::error::unsupported_address); } diff --git a/src/net/parse.h b/src/net/parse.h index 9f0d66ea6..0d8fda711 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -36,6 +36,8 @@ namespace net { + void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port); + /*! Identifies onion, i2p and IPv4 addresses and returns them as a generic `network_address`. If the type is unsupported, it might be a hostname, diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index a44297c17..bb51be242 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -108,10 +108,11 @@ namespace namespace nodetool { - const command_line::arg_descriptor<std::string> arg_p2p_bind_ip = {"p2p-bind-ip", "Interface for p2p network protocol", "0.0.0.0"}; + const command_line::arg_descriptor<std::string> arg_p2p_bind_ip = {"p2p-bind-ip", "Interface for p2p network protocol (IPv4)", "0.0.0.0"}; + const command_line::arg_descriptor<std::string> arg_p2p_bind_ipv6_address = {"p2p-bind-ipv6-address", "Interface for p2p network protocol (IPv6)", "::"}; const command_line::arg_descriptor<std::string, false, true, 2> arg_p2p_bind_port = { "p2p-bind-port" - , "Port for p2p network protocol" + , "Port for p2p network protocol (IPv4)" , std::to_string(config::P2P_DEFAULT_PORT) , {{ &cryptonote::arg_testnet_on, &cryptonote::arg_stagenet_on }} , [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { @@ -122,6 +123,20 @@ namespace nodetool return val; } }; + const command_line::arg_descriptor<std::string, false, true, 2> arg_p2p_bind_port_ipv6 = { + "p2p-bind-port-ipv6" + , "Port for p2p network protocol (IPv6)" + , std::to_string(config::P2P_DEFAULT_PORT) + , {{ &cryptonote::arg_testnet_on, &cryptonote::arg_stagenet_on }} + , [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { + if (testnet_stagenet[0] && defaulted) + return std::to_string(config::testnet::P2P_DEFAULT_PORT); + else if (testnet_stagenet[1] && defaulted) + return std::to_string(config::stagenet::P2P_DEFAULT_PORT); + return val; + } + }; + const command_line::arg_descriptor<uint32_t> arg_p2p_external_port = {"p2p-external-port", "External port for p2p network protocol (if port forwarding used with NAT)", 0}; const command_line::arg_descriptor<bool> arg_p2p_allow_local_ip = {"allow-local-ip", "Allow local ip add to peer list, mostly in debug purposes"}; const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_add_peer = {"add-peer", "Manually add peer to local peerlist"}; @@ -136,6 +151,8 @@ namespace nodetool const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; const command_line::arg_descriptor<std::string> arg_igd = {"igd", "UPnP port mapping (disabled, enabled, delayed)", "delayed"}; + const command_line::arg_descriptor<bool> arg_p2p_use_ipv6 = {"p2p-use-ipv6", "Enable IPv6 for p2p", false}; + const command_line::arg_descriptor<bool> arg_p2p_require_ipv4 = {"p2p-require-ipv4", "Require successful IPv4 bind for p2p", true}; const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max number of out peers", -1}; const command_line::arg_descriptor<int64_t> arg_in_peers = {"in-peers", "set max number of in peers", -1}; const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1}; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 0d95ceb99..6d2ae878f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -151,7 +151,9 @@ namespace nodetool : m_connect(nullptr), m_net_server(epee::net_utils::e_connection_type_P2P), m_bind_ip(), + m_bind_ipv6_address(), m_port(), + m_port_ipv6(), m_our_address(), m_peerlist(), m_config{}, @@ -167,7 +169,9 @@ namespace nodetool : m_connect(nullptr), m_net_server(public_service, epee::net_utils::e_connection_type_P2P), m_bind_ip(), + m_bind_ipv6_address(), m_port(), + m_port_ipv6(), m_our_address(), m_peerlist(), m_config{}, @@ -182,7 +186,9 @@ namespace nodetool connect_func* m_connect; net_server m_net_server; std::string m_bind_ip; + std::string m_bind_ipv6_address; std::string m_port; + std::string m_port_ipv6; epee::net_utils::network_address m_our_address; // in anonymity networks peerlist_manager m_peerlist; config m_config; @@ -248,6 +254,7 @@ namespace nodetool size_t get_public_white_peers_count(); size_t get_public_gray_peers_count(); void get_public_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white); + void get_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white); size_t get_zone_count() const { return m_network_zones.size(); } void change_max_out_public_peers(size_t count); @@ -357,7 +364,13 @@ namespace nodetool bool is_peer_used(const peerlist_entry& peer); bool is_peer_used(const anchor_peerlist_entry& peer); bool is_addr_connected(const epee::net_utils::network_address& peer); - void add_upnp_port_mapping(uint32_t port); + void add_upnp_port_mapping_impl(uint32_t port, bool ipv6=false); + void add_upnp_port_mapping_v4(uint32_t port); + void add_upnp_port_mapping_v6(uint32_t port); + void add_upnp_port_mapping(uint32_t port, bool ipv4=true, bool ipv6=false); + void delete_upnp_port_mapping_impl(uint32_t port, bool ipv6=false); + void delete_upnp_port_mapping_v4(uint32_t port); + void delete_upnp_port_mapping_v6(uint32_t port); void delete_upnp_port_mapping(uint32_t port); template<class t_callback> bool try_ping(basic_node_data& node_data, p2p_connection_context& context, const t_callback &cb); @@ -419,12 +432,15 @@ namespace nodetool bool m_have_address; bool m_first_connection_maker_call; uint32_t m_listening_port; + uint32_t m_listening_port_ipv6; uint32_t m_external_port; uint16_t m_rpc_port; bool m_allow_local_ip; bool m_hide_my_port; igd_t m_igd; bool m_offline; + bool m_use_ipv6; + bool m_require_ipv4; std::atomic<bool> is_closing; std::unique_ptr<boost::thread> mPeersLoggerThread; //critical_section m_connections_lock; @@ -485,7 +501,11 @@ namespace nodetool const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s const int64_t default_limit_down = P2P_DEFAULT_LIMIT_RATE_DOWN; // kB/s extern const command_line::arg_descriptor<std::string> arg_p2p_bind_ip; + extern const command_line::arg_descriptor<std::string> arg_p2p_bind_ipv6_address; extern const command_line::arg_descriptor<std::string, false, true, 2> arg_p2p_bind_port; + extern const command_line::arg_descriptor<std::string, false, true, 2> arg_p2p_bind_port_ipv6; + extern const command_line::arg_descriptor<bool> arg_p2p_use_ipv6; + extern const command_line::arg_descriptor<bool> arg_p2p_require_ipv4; extern const command_line::arg_descriptor<uint32_t> arg_p2p_external_port; extern const command_line::arg_descriptor<bool> arg_p2p_allow_local_ip; extern const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_add_peer; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 98484fe78..8c0cff7e2 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -93,7 +93,11 @@ namespace nodetool void node_server<t_payload_net_handler>::init_options(boost::program_options::options_description& desc) { command_line::add_arg(desc, arg_p2p_bind_ip); + command_line::add_arg(desc, arg_p2p_bind_ipv6_address); command_line::add_arg(desc, arg_p2p_bind_port, false); + command_line::add_arg(desc, arg_p2p_bind_port_ipv6, false); + command_line::add_arg(desc, arg_p2p_use_ipv6); + command_line::add_arg(desc, arg_p2p_require_ipv4); command_line::add_arg(desc, arg_p2p_external_port); command_line::add_arg(desc, arg_p2p_allow_local_ip); command_line::add_arg(desc, arg_p2p_add_peer); @@ -341,7 +345,9 @@ namespace nodetool network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; public_zone.m_connect = &public_connect; public_zone.m_bind_ip = command_line::get_arg(vm, arg_p2p_bind_ip); + public_zone.m_bind_ipv6_address = command_line::get_arg(vm, arg_p2p_bind_ipv6_address); public_zone.m_port = command_line::get_arg(vm, arg_p2p_bind_port); + public_zone.m_port_ipv6 = command_line::get_arg(vm, arg_p2p_bind_port_ipv6); public_zone.m_can_pingback = true; m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); @@ -375,6 +381,8 @@ namespace nodetool return false; } m_offline = command_line::get_arg(vm, cryptonote::arg_offline); + m_use_ipv6 = command_line::get_arg(vm, arg_p2p_use_ipv6); + m_require_ipv4 = command_line::get_arg(vm, arg_p2p_require_ipv4); if (command_line::has_arg(vm, arg_p2p_add_peer)) { @@ -518,12 +526,17 @@ namespace nodetool std::string host = addr; std::string port = std::to_string(default_port); - size_t pos = addr.find_last_of(':'); - if (std::string::npos != pos) + size_t colon_pos = addr.find_last_of(':'); + size_t dot_pos = addr.find_last_of('.'); + size_t square_brace_pos = addr.find('['); + + // IPv6 will have colons regardless. IPv6 and IPv4 address:port will have a colon but also either a . or a [ + // as IPv6 addresses specified as address:port are to be specified as "[addr:addr:...:addr]:port" + // One may also specify an IPv6 address as simply "[addr:addr:...:addr]" without the port; in that case + // the square braces will be stripped here. + if ((std::string::npos != colon_pos && std::string::npos != dot_pos) || std::string::npos != square_brace_pos) { - CHECK_AND_ASSERT_MES(addr.length() - 1 != pos && 0 != pos, false, "Failed to parse seed address from string: '" << addr << '\''); - host = addr.substr(0, pos); - port = addr.substr(pos + 1); + net::get_network_address_host_and_port(addr, host, port); } MINFO("Resolving node address: host=" << host << ", port=" << port); @@ -546,7 +559,9 @@ namespace nodetool } else { - MWARNING("IPv6 unsupported, skip '" << host << "' -> " << endpoint.address().to_v6().to_string(ec)); + epee::net_utils::network_address na{epee::net_utils::ipv6_network_address{endpoint.address().to_v6(), endpoint.port()}}; + seed_nodes.push_back(na); + MINFO("Added node: " << na.str()); } } return true; @@ -780,21 +795,40 @@ namespace nodetool if (!zone.second.m_bind_ip.empty()) { + std::string ipv6_addr = ""; + std::string ipv6_port = ""; zone.second.m_net_server.set_connection_filter(this); - MINFO("Binding on " << zone.second.m_bind_ip << ":" << zone.second.m_port); - res = zone.second.m_net_server.init_server(zone.second.m_port, zone.second.m_bind_ip, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + MINFO("Binding (IPv4) on " << zone.second.m_bind_ip << ":" << zone.second.m_port); + if (!zone.second.m_bind_ipv6_address.empty() && m_use_ipv6) + { + ipv6_addr = zone.second.m_bind_ipv6_address; + ipv6_port = zone.second.m_port_ipv6; + MINFO("Binding (IPv6) on " << zone.second.m_bind_ipv6_address << ":" << zone.second.m_port_ipv6); + } + res = zone.second.m_net_server.init_server(zone.second.m_port, zone.second.m_bind_ip, ipv6_port, ipv6_addr, m_use_ipv6, m_require_ipv4, epee::net_utils::ssl_support_t::e_ssl_support_disabled); CHECK_AND_ASSERT_MES(res, false, "Failed to bind server"); } } m_listening_port = public_zone.m_net_server.get_binded_port(); - MLOG_GREEN(el::Level::Info, "Net service bound to " << public_zone.m_bind_ip << ":" << m_listening_port); + MLOG_GREEN(el::Level::Info, "Net service bound (IPv4) to " << public_zone.m_bind_ip << ":" << m_listening_port); + if (m_use_ipv6) + { + m_listening_port_ipv6 = public_zone.m_net_server.get_binded_port_ipv6(); + MLOG_GREEN(el::Level::Info, "Net service bound (IPv6) to " << public_zone.m_bind_ipv6_address << ":" << m_listening_port_ipv6); + } if(m_external_port) MDEBUG("External port defined as " << m_external_port); // add UPnP port mapping if(m_igd == igd) - add_upnp_port_mapping(m_listening_port); + { + add_upnp_port_mapping_v4(m_listening_port); + if (m_use_ipv6) + { + add_upnp_port_mapping_v6(m_listening_port_ipv6); + } + } return res; } @@ -1705,6 +1739,15 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::get_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white) + { + for (auto &zone: m_network_zones) + { + zone.second.m_peerlist.get_peerlist(gray, white); // appends + } + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::idle_worker() { m_peer_handshake_idle_maker_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::peer_sync_idle_maker, this)); @@ -1997,19 +2040,43 @@ namespace nodetool if(!node_data.my_port) return false; - CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), false, - "Only IPv4 addresses are supported here"); + bool address_ok = (context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id() || context.m_remote_address.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id()); + CHECK_AND_ASSERT_MES(address_ok, false, + "Only IPv4 or IPv6 addresses are supported here"); const epee::net_utils::network_address na = context.m_remote_address; - uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip(); + std::string ip; + uint32_t ipv4_addr; + boost::asio::ip::address_v6 ipv6_addr; + bool is_ipv4; + if (na.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + ipv4_addr = na.as<const epee::net_utils::ipv4_network_address>().ip(); + ip = epee::string_tools::get_ip_string_from_int32(ipv4_addr); + is_ipv4 = true; + } + else + { + ipv6_addr = na.as<const epee::net_utils::ipv6_network_address>().ip(); + ip = ipv6_addr.to_string(); + is_ipv4 = false; + } network_zone& zone = m_network_zones.at(na.get_zone()); if(!zone.m_peerlist.is_host_allowed(context.m_remote_address)) return false; - std::string ip = epee::string_tools::get_ip_string_from_int32(actual_ip); std::string port = epee::string_tools::num_to_string_fast(node_data.my_port); - epee::net_utils::network_address address{epee::net_utils::ipv4_network_address(actual_ip, node_data.my_port)}; + + epee::net_utils::network_address address; + if (is_ipv4) + { + address = epee::net_utils::network_address{epee::net_utils::ipv4_network_address(ipv4_addr, node_data.my_port)}; + } + else + { + address = epee::net_utils::network_address{epee::net_utils::ipv6_network_address(ipv6_addr, node_data.my_port)}; + } peerid_type pr = node_data.peer_id; bool r = zone.m_net_server.connect_async(ip, port, zone.m_config.m_net_config.ping_connection_timeout, [cb, /*context,*/ address, pr, this]( const typename net_server::t_connection_context& ping_context, @@ -2193,12 +2260,19 @@ namespace nodetool //try ping to be sure that we can add this peer to peer_list try_ping(arg.node_data, context, [peer_id_l, port_l, context, this]() { - CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), void(), - "Only IPv4 addresses are supported here"); + CHECK_AND_ASSERT_MES((context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id() || context.m_remote_address.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id()), void(), + "Only IPv4 or IPv6 addresses are supported here"); //called only(!) if success pinged, update local peerlist peerlist_entry pe; const epee::net_utils::network_address na = context.m_remote_address; - pe.adr = epee::net_utils::ipv4_network_address(na.as<epee::net_utils::ipv4_network_address>().ip(), port_l); + if (context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + pe.adr = epee::net_utils::ipv4_network_address(na.as<epee::net_utils::ipv4_network_address>().ip(), port_l); + } + else + { + pe.adr = epee::net_utils::ipv6_network_address(na.as<epee::net_utils::ipv6_network_address>().ip(), port_l); + } time_t last_seen; time(&last_seen); pe.last_seen = static_cast<int64_t>(last_seen); @@ -2367,10 +2441,11 @@ namespace nodetool auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); if (public_zone != m_network_zones.end()) { - const auto current = public_zone->second.m_config.m_net_config.max_out_connection_count; + const auto current = public_zone->second.m_net_server.get_config_object().get_out_connections_count(); public_zone->second.m_config.m_net_config.max_out_connection_count = count; if(current > count) public_zone->second.m_net_server.get_config_object().del_out_connections(current - count); + m_payload_handler.set_max_out_peers(count); } } @@ -2389,7 +2464,7 @@ namespace nodetool auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); if (public_zone != m_network_zones.end()) { - const auto current = public_zone->second.m_config.m_net_config.max_in_connection_count; + const auto current = public_zone->second.m_net_server.get_config_object().get_in_connections_count(); public_zone->second.m_config.m_net_config.max_in_connection_count = count; if(current > count) public_zone->second.m_net_server.get_config_object().del_in_connections(current - count); @@ -2565,16 +2640,19 @@ namespace nodetool } template<class t_payload_net_handler> - void node_server<t_payload_net_handler>::add_upnp_port_mapping(uint32_t port) + void node_server<t_payload_net_handler>::add_upnp_port_mapping_impl(uint32_t port, bool ipv6) // if ipv6 false, do ipv4 { - MDEBUG("Attempting to add IGD port mapping."); + std::string ipversion = ipv6 ? "(IPv6)" : "(IPv4)"; + MDEBUG("Attempting to add IGD port mapping " << ipversion << "."); int result; + const int ipv6_arg = ipv6 ? 1 : 0; + #if MINIUPNPC_API_VERSION > 13 // default according to miniupnpc.h unsigned char ttl = 2; - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, ttl, &result); + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, ipv6_arg, ttl, &result); #else - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, ipv6_arg, &result); #endif UPNPUrls urls; IGDdatas igdData; @@ -2611,16 +2689,38 @@ namespace nodetool } template<class t_payload_net_handler> - void node_server<t_payload_net_handler>::delete_upnp_port_mapping(uint32_t port) + void node_server<t_payload_net_handler>::add_upnp_port_mapping_v4(uint32_t port) { - MDEBUG("Attempting to delete IGD port mapping."); + add_upnp_port_mapping_impl(port, false); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::add_upnp_port_mapping_v6(uint32_t port) + { + add_upnp_port_mapping_impl(port, true); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::add_upnp_port_mapping(uint32_t port, bool ipv4, bool ipv6) + { + if (ipv4) add_upnp_port_mapping_v4(port); + if (ipv6) add_upnp_port_mapping_v6(port); + } + + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::delete_upnp_port_mapping_impl(uint32_t port, bool ipv6) + { + std::string ipversion = ipv6 ? "(IPv6)" : "(IPv4)"; + MDEBUG("Attempting to delete IGD port mapping " << ipversion << "."); int result; + const int ipv6_arg = ipv6 ? 1 : 0; #if MINIUPNPC_API_VERSION > 13 // default according to miniupnpc.h unsigned char ttl = 2; - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, ttl, &result); + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, ipv6_arg, ttl, &result); #else - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, ipv6_arg, &result); #endif UPNPUrls urls; IGDdatas igdData; @@ -2653,6 +2753,25 @@ namespace nodetool } } + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::delete_upnp_port_mapping_v4(uint32_t port) + { + delete_upnp_port_mapping_impl(port, false); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::delete_upnp_port_mapping_v6(uint32_t port) + { + delete_upnp_port_mapping_impl(port, true); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::delete_upnp_port_mapping(uint32_t port) + { + delete_upnp_port_mapping_v4(port); + delete_upnp_port_mapping_v6(port); + } + template<typename t_payload_net_handler> boost::optional<p2p_connection_context_t<typename t_payload_net_handler::connection_context>> node_server<t_payload_net_handler>::socks_connect(network_zone& zone, const epee::net_utils::network_address& remote, epee::net_utils::ssl_support_t ssl_support) @@ -2671,13 +2790,34 @@ namespace nodetool boost::optional<p2p_connection_context_t<typename t_payload_net_handler::connection_context>> node_server<t_payload_net_handler>::public_connect(network_zone& zone, epee::net_utils::network_address const& na, epee::net_utils::ssl_support_t ssl_support) { - CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), boost::none, - "Only IPv4 addresses are supported here"); - const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); + bool is_ipv4 = na.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(); + bool is_ipv6 = na.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id(); + CHECK_AND_ASSERT_MES(is_ipv4 || is_ipv6, boost::none, + "Only IPv4 or IPv6 addresses are supported here"); + + std::string address; + std::string port; + + if (is_ipv4) + { + const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); + address = epee::string_tools::get_ip_string_from_int32(ipv4.ip()); + port = epee::string_tools::num_to_string_fast(ipv4.port()); + } + else if (is_ipv6) + { + const epee::net_utils::ipv6_network_address &ipv6 = na.as<const epee::net_utils::ipv6_network_address>(); + address = ipv6.ip().to_string(); + port = epee::string_tools::num_to_string_fast(ipv6.port()); + } + else + { + LOG_ERROR("Only IPv4 or IPv6 addresses are supported here"); + return boost::none; + } typename net_server::t_connection_context con{}; - const bool res = zone.m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), - epee::string_tools::num_to_string_fast(ipv4.port()), + const bool res = zone.m_net_server.connect(address, port, zone.m_config.m_net_config.connection_timeout, con, "0.0.0.0", ssl_support); diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index 883997fd6..c65b9dd82 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -278,6 +278,9 @@ namespace nodetool // was moved to the gray list (if it's not accessibe, which the attacker can check if // the address accepts incoming connections) or it was the oldest to still fit in the 250 items, // so its last_seen is old. + // + // See Cao, Tong et al. "Exploring the Monero Peer-to-Peer Network". https://eprint.iacr.org/2019/411 + // const uint32_t pick_depth = anonymize ? depth + depth / 5 : depth; bs_head.reserve(pick_depth); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) @@ -290,7 +293,7 @@ namespace nodetool if (anonymize) { - std::random_shuffle(bs_head.begin(), bs_head.end()); + std::shuffle(bs_head.begin(), bs_head.end(), crypto::random_device{}); if (bs_head.size() > depth) bs_head.resize(depth); for (auto &e: bs_head) diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 32f30adca..05eb36e65 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -76,6 +76,9 @@ namespace boost case epee::net_utils::ipv4_network_address::get_type_id(): do_serialize<epee::net_utils::ipv4_network_address>(is_saving, a, na); break; + case epee::net_utils::ipv6_network_address::get_type_id(): + do_serialize<epee::net_utils::ipv6_network_address>(is_saving, a, na); + break; case net::tor_address::get_type_id(): do_serialize<net::tor_address>(is_saving, a, na); break; @@ -99,6 +102,34 @@ namespace boost } template <class Archive, class ver_type> + inline void serialize(Archive &a, boost::asio::ip::address_v6& v6, const ver_type ver) + { + if (typename Archive::is_saving()) + { + auto bytes = v6.to_bytes(); + for (auto &e: bytes) a & e; + } + else + { + boost::asio::ip::address_v6::bytes_type bytes; + for (auto &e: bytes) a & e; + v6 = boost::asio::ip::address_v6(bytes); + } + } + + template <class Archive, class ver_type> + inline void serialize(Archive &a, epee::net_utils::ipv6_network_address& na, const ver_type ver) + { + boost::asio::ip::address_v6 ip{na.ip()}; + uint16_t port{na.port()}; + a & ip; + a & port; + if (!typename Archive::is_saving()) + na = epee::net_utils::ipv6_network_address{ip, port}; + } + + + template <class Archive, class ver_type> inline void save(Archive& a, const net::tor_address& na, const ver_type) { const size_t length = std::strlen(na.host_str()); diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index cffe8e1eb..06577d37e 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -47,7 +47,7 @@ set(rpc_base_headers rpc_args.h) set(rpc_headers - rpc_handler.cpp) + rpc_handler.h) set(daemon_rpc_server_headers) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index d80219f1c..73138686d 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -142,6 +142,7 @@ namespace cryptonote { m_restricted = restricted; m_net_server.set_threads_prefix("RPC"); + m_net_server.set_connection_filter(&m_p2p); auto rpc_config = cryptonote::rpc_args::process(vm, true); if (!rpc_config) @@ -161,7 +162,9 @@ namespace cryptonote auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; return epee::http_server_impl_base<core_rpc_server, connection_context>::init( - rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) + rng, std::move(port), std::move(rpc_config->bind_ip), + std::move(rpc_config->bind_ipv6_address), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4), + std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -173,6 +176,24 @@ namespace cryptonote } return true; } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::add_host_fail(const connection_context *ctx) + { + if(!ctx || !ctx->m_remote_address.is_blockable()) + return false; + + CRITICAL_REGION_LOCAL(m_host_fails_score_lock); + uint64_t fails = ++m_host_fails_score[ctx->m_remote_address.host_str()]; + MDEBUG("Host " << ctx->m_remote_address.host_str() << " fail score=" << fails); + if(fails > RPC_IP_FAILS_BEFORE_BLOCK) + { + auto it = m_host_fails_score.find(ctx->m_remote_address.host_str()); + CHECK_AND_ASSERT_MES(it != m_host_fails_score.end(), false, "internal error"); + it->second = RPC_IP_FAILS_BEFORE_BLOCK/2; + m_p2p.block_host(ctx->m_remote_address); + } + return true; + } #define CHECK_CORE_READY() do { if(!check_core_ready()){res.status = CORE_RPC_STATUS_BUSY;return true;} } while(0) //------------------------------------------------------------------------------------------------------------------------------ @@ -300,6 +321,7 @@ namespace cryptonote if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; + add_host_fail(ctx); return false; } @@ -423,6 +445,7 @@ namespace cryptonote if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, res.m_block_ids, res.start_height, res.current_height, false)) { res.status = "Failed"; + add_host_fail(ctx); return false; } @@ -603,7 +626,8 @@ namespace cryptonote return true; } const cryptonote::blobdata pruned = ss.str(); - sorted_txs.push_back(std::make_tuple(h, pruned, get_transaction_prunable_hash(tx), std::string(i->tx_blob, pruned.size()))); + const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx); + sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); @@ -862,6 +886,8 @@ namespace cryptonote add_reason(reason, "fee too low"); if ((res.not_rct = tvc.m_not_rct)) add_reason(reason, "tx is not ringct"); + if ((res.too_few_outputs = tvc.m_too_few_outputs)) + add_reason(reason, "too few outputs"); const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { @@ -1018,24 +1044,36 @@ namespace cryptonote PERF_TIMER(on_get_peer_list); std::vector<nodetool::peerlist_entry> white_list; std::vector<nodetool::peerlist_entry> gray_list; - m_p2p.get_public_peerlist(gray_list, white_list); - res.white_list.reserve(white_list.size()); + if (req.public_only) + { + m_p2p.get_public_peerlist(gray_list, white_list); + } + else + { + m_p2p.get_peerlist(gray_list, white_list); + } + for (auto & entry : white_list) { if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); + else if (entry.adr.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id()) + res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv6_network_address>().host_str(), + entry.adr.as<epee::net_utils::ipv6_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); else res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed, entry.rpc_port); } - res.gray_list.reserve(gray_list.size()); for (auto & entry : gray_list) { if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); + else if (entry.adr.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id()) + res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv6_network_address>().host_str(), + entry.adr.as<epee::net_utils::ipv6_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); else res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed, entry.rpc_port); } @@ -1044,6 +1082,45 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_public_nodes(const COMMAND_RPC_GET_PUBLIC_NODES::request& req, COMMAND_RPC_GET_PUBLIC_NODES::response& res, const connection_context *ctx) + { + PERF_TIMER(on_get_public_nodes); + + COMMAND_RPC_GET_PEER_LIST::response peer_list_res; + const bool success = on_get_peer_list(COMMAND_RPC_GET_PEER_LIST::request(), peer_list_res, ctx); + res.status = peer_list_res.status; + if (!success) + { + return false; + } + if (res.status != CORE_RPC_STATUS_OK) + { + return true; + } + + const auto collect = [](const std::vector<peer> &peer_list, std::vector<public_node> &public_nodes) + { + for (const auto &entry : peer_list) + { + if (entry.rpc_port != 0) + { + public_nodes.emplace_back(entry); + } + } + }; + + if (req.white) + { + collect(peer_list_res.white_list, res.white); + } + if (req.gray) + { + collect(peer_list_res.gray_list, res.gray); + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_hash_rate); @@ -1256,6 +1333,20 @@ namespace cryptonote return false; } + if(req.reserve_size && !req.extra_nonce.empty()) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Cannot specify both a reserve_size and an extra_nonce"; + return false; + } + + if(req.extra_nonce.size() > 510) + { + error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_RESERVE_SIZE; + error_resp.message = "Too big extra_nonce size, maximum 510 hex chars"; + return false; + } + cryptonote::address_parse_info info; if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, nettype(), req.wallet_address)) @@ -1273,7 +1364,17 @@ namespace cryptonote block b; cryptonote::blobdata blob_reserve; - blob_reserve.resize(req.reserve_size, 0); + if(!req.extra_nonce.empty()) + { + if(!string_tools::parse_hexstr_to_binbuff(req.extra_nonce, blob_reserve)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Parameter extra_nonce should be a hex string"; + return false; + } + } + else + blob_reserve.resize(req.reserve_size, 0); cryptonote::difficulty_type wdiff; crypto::hash prev_block; if (!req.prev_block.empty()) @@ -1600,38 +1701,55 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH>(invoke_http_mode::JON_RPC, "getblockheaderbyhash", req, res, r)) return r; - crypto::hash block_hash; - bool hash_parsed = parse_hash256(req.hash, block_hash); - if(!hash_parsed) - { - error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; - error_resp.message = "Failed to parse hex representation of block hash. Hex = " + req.hash + '.'; - return false; - } - block blk; - bool orphan = false; - bool have_block = m_core.get_block_by_hash(block_hash, blk, &orphan); - if (!have_block) - { - error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Internal error: can't get block by hash. Hash = " + req.hash + '.'; - return false; - } - if (blk.miner_tx.vin.size() != 1 || blk.miner_tx.vin.front().type() != typeid(txin_gen)) + auto get = [this](const std::string &hash, bool fill_pow_hash, block_header_response &block_header, bool restricted, epee::json_rpc::error& error_resp) -> bool { + crypto::hash block_hash; + bool hash_parsed = parse_hash256(hash, block_hash); + if(!hash_parsed) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Failed to parse hex representation of block hash. Hex = " + hash + '.'; + return false; + } + block blk; + bool orphan = false; + bool have_block = m_core.get_block_by_hash(block_hash, blk, &orphan); + if (!have_block) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: can't get block by hash. Hash = " + hash + '.'; + return false; + } + if (blk.miner_tx.vin.size() != 1 || blk.miner_tx.vin.front().type() != typeid(txin_gen)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: coinbase transaction in the block has the wrong type"; + return false; + } + uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, block_header, fill_pow_hash && !restricted); + if (!response_filled) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: can't produce valid response."; + return false; + } + return true; + }; + + const bool restricted = m_restricted && ctx; + if (!req.hash.empty()) { - error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Internal error: coinbase transaction in the block has the wrong type"; - return false; + if (!get(req.hash, req.fill_pow_hash, res.block_header, restricted, error_resp)) + return false; } - uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - const bool restricted = m_restricted && ctx; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); - if (!response_filled) + res.block_headers.reserve(req.hashes.size()); + for (const std::string &hash: req.hashes) { - error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Internal error: can't produce valid response."; - return false; + res.block_headers.push_back({}); + if (!get(hash, req.fill_pow_hash, res.block_headers.back(), restricted, error_resp)) + return false; } + res.status = CORE_RPC_STATUS_OK; return true; } @@ -2061,7 +2179,7 @@ namespace cryptonote PERF_TIMER(on_get_alternate_chains); try { - std::list<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains(); + std::vector<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains(); for (const auto &i: chains) { difficulty_type wdiff = i.first.cumulative_difficulty; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 663975617..e91d4c953 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -108,6 +108,7 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/mining_status", on_mining_status, COMMAND_RPC_MINING_STATUS, !m_restricted) MAP_URI_AUTO_JON2_IF("/save_bc", on_save_bc, COMMAND_RPC_SAVE_BC, !m_restricted) MAP_URI_AUTO_JON2_IF("/get_peer_list", on_get_peer_list, COMMAND_RPC_GET_PEER_LIST, !m_restricted) + MAP_URI_AUTO_JON2_IF("/get_public_nodes", on_get_public_nodes, COMMAND_RPC_GET_PUBLIC_NODES, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_hash_rate", on_set_log_hash_rate, COMMAND_RPC_SET_LOG_HASH_RATE, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted) @@ -186,6 +187,7 @@ namespace cryptonote bool on_get_net_stats(const COMMAND_RPC_GET_NET_STATS::request& req, COMMAND_RPC_GET_NET_STATS::response& res, const connection_context *ctx = NULL); bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, const connection_context *ctx = NULL); bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res, const connection_context *ctx = NULL); + bool on_get_public_nodes(const COMMAND_RPC_GET_PUBLIC_NODES::request& req, COMMAND_RPC_GET_PUBLIC_NODES::response& res, const connection_context *ctx = NULL); bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx = NULL); bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res, const connection_context *ctx = NULL); bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res, const connection_context *ctx = NULL); @@ -236,6 +238,7 @@ namespace cryptonote private: bool check_core_busy(); bool check_core_ready(); + bool add_host_fail(const connection_context *ctx); //utils uint64_t get_block_reward(const block& blk); @@ -256,6 +259,8 @@ private: bool m_was_bootstrap_ever_used; network_type m_nettype; bool m_restricted; + epee::critical_section m_host_fails_score_lock; + std::map<std::string, uint64_t> m_host_fails_score; }; } diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 571a71207..aed967efb 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -87,7 +87,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 7 +#define CORE_RPC_VERSION_MINOR 9 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -612,6 +612,7 @@ namespace cryptonote bool overspend; bool fee_too_low; bool not_rct; + bool too_few_outputs; bool sanity_check_failed; bool untrusted; @@ -627,6 +628,7 @@ namespace cryptonote KV_SERIALIZE(overspend) KV_SERIALIZE(fee_too_low) KV_SERIALIZE(not_rct) + KV_SERIALIZE(too_few_outputs) KV_SERIALIZE(sanity_check_failed) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() @@ -922,11 +924,13 @@ namespace cryptonote uint64_t reserve_size; //max 255 bytes std::string wallet_address; std::string prev_block; + std::string extra_nonce; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(reserve_size) KV_SERIALIZE(wallet_address) KV_SERIALIZE(prev_block) + KV_SERIALIZE(extra_nonce) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1094,10 +1098,12 @@ namespace cryptonote struct request_t { std::string hash; + std::vector<std::string> hashes; bool fill_pow_hash; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(hash) + KV_SERIALIZE(hashes) KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; @@ -1107,10 +1113,12 @@ namespace cryptonote { std::string status; block_header_response block_header; + std::vector<block_header_response> block_headers; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block_header) + KV_SERIALIZE(block_headers) KV_SERIALIZE(status) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() @@ -1200,6 +1208,9 @@ namespace cryptonote peer(uint64_t id, const std::string &host, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) : id(id), host(host), ip(0), port(0), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) {} + peer(uint64_t id, const std::string &host, uint16_t port, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) + : id(id), host(host), ip(0), port(port), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) + {} peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) : id(id), host(epee::string_tools::get_ip_string_from_int32(ip)), ip(ip), port(port), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) {} @@ -1219,7 +1230,10 @@ namespace cryptonote { struct request_t { + bool public_only; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(public_only, true) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1239,6 +1253,54 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct public_node + { + std::string host; + uint64_t last_seen; + uint16_t rpc_port; + + public_node() = delete; + + public_node(const peer &peer) + : host(peer.host), last_seen(peer.last_seen), rpc_port(peer.rpc_port) + {} + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(host) + KV_SERIALIZE(last_seen) + KV_SERIALIZE(rpc_port) + END_KV_SERIALIZE_MAP() + }; + + struct COMMAND_RPC_GET_PUBLIC_NODES + { + struct request_t + { + bool gray; + bool white; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(gray, false) + KV_SERIALIZE_OPT(white, true) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string status; + std::vector<public_node> gray; + std::vector<public_node> white; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(gray) + KV_SERIALIZE(white) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_SET_LOG_HASH_RATE { struct request_t @@ -2095,7 +2157,7 @@ namespace cryptonote struct response_t { std::string status; - std::list<chain_info> chains; + std::vector<chain_info> chains; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 612b2cab6..890380dc8 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -343,6 +343,11 @@ namespace rpc if (!res.error_details.empty()) res.error_details += " and "; res.error_details = "tx is not ringct"; } + if (tvc.m_too_few_outputs) + { + if (!res.error_details.empty()) res.error_details += " and "; + res.error_details = "too few outputs"; + } if (res.error_details.empty()) { res.error_details = "an unknown issue was found with the transaction"; diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index 4479bd1f1..68b33cb8c 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -90,6 +90,9 @@ namespace cryptonote rpc_args::descriptors::descriptors() : rpc_bind_ip({"rpc-bind-ip", rpc_args::tr("Specify IP to bind RPC server"), "127.0.0.1"}) + , rpc_bind_ipv6_address({"rpc-bind-ipv6-address", rpc_args::tr("Specify IPv6 address to bind RPC server"), "::1"}) + , rpc_use_ipv6({"rpc-use-ipv6", rpc_args::tr("Allow IPv6 for RPC"), false}) + , rpc_require_ipv4({"rpc-require-ipv4", rpc_args::tr("Require successful IPv4 bind for RPC"), true}) , rpc_login({"rpc-login", rpc_args::tr("Specify username[:password] required for RPC server"), "", true}) , confirm_external_bind({"confirm-external-bind", rpc_args::tr("Confirm rpc-bind-ip value is NOT a loopback (local) IP")}) , rpc_access_control_origins({"rpc-access-control-origins", rpc_args::tr("Specify a comma separated list of origins to allow cross origin resource sharing"), ""}) @@ -108,6 +111,9 @@ namespace cryptonote { const descriptors arg{}; command_line::add_arg(desc, arg.rpc_bind_ip); + command_line::add_arg(desc, arg.rpc_bind_ipv6_address); + command_line::add_arg(desc, arg.rpc_use_ipv6); + command_line::add_arg(desc, arg.rpc_require_ipv4); command_line::add_arg(desc, arg.rpc_login); command_line::add_arg(desc, arg.confirm_external_bind); command_line::add_arg(desc, arg.rpc_access_control_origins); @@ -127,6 +133,9 @@ namespace cryptonote rpc_args config{}; config.bind_ip = command_line::get_arg(vm, arg.rpc_bind_ip); + config.bind_ipv6_address = command_line::get_arg(vm, arg.rpc_bind_ipv6_address); + config.use_ipv6 = command_line::get_arg(vm, arg.rpc_use_ipv6); + config.require_ipv4 = command_line::get_arg(vm, arg.rpc_require_ipv4); if (!config.bind_ip.empty()) { // always parse IP here for error consistency @@ -148,6 +157,34 @@ namespace cryptonote return boost::none; } } + if (!config.bind_ipv6_address.empty()) + { + // allow square braces, but remove them here if present + if (config.bind_ipv6_address.find('[') != std::string::npos) + { + config.bind_ipv6_address = config.bind_ipv6_address.substr(1, config.bind_ipv6_address.size() - 2); + } + + + // always parse IP here for error consistency + boost::system::error_code ec{}; + const auto parsed_ip = boost::asio::ip::address::from_string(config.bind_ipv6_address, ec); + if (ec) + { + LOG_ERROR(tr("Invalid IP address given for --") << arg.rpc_bind_ipv6_address.name); + return boost::none; + } + + if (!parsed_ip.is_loopback() && !command_line::get_arg(vm, arg.confirm_external_bind)) + { + LOG_ERROR( + "--" << arg.rpc_bind_ipv6_address.name << + tr(" permits inbound unencrypted external connections. Consider SSH tunnel or SSL proxy instead. Override with --") << + arg.confirm_external_bind.name + ); + return boost::none; + } + } const char *env_rpc_login = nullptr; const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); diff --git a/src/rpc/rpc_args.h b/src/rpc/rpc_args.h index 619f02b42..cd154a4d0 100644 --- a/src/rpc/rpc_args.h +++ b/src/rpc/rpc_args.h @@ -52,6 +52,9 @@ namespace cryptonote descriptors& operator=(descriptors&&) = delete; const command_line::arg_descriptor<std::string> rpc_bind_ip; + const command_line::arg_descriptor<std::string> rpc_bind_ipv6_address; + const command_line::arg_descriptor<bool> rpc_use_ipv6; + const command_line::arg_descriptor<bool> rpc_require_ipv4; const command_line::arg_descriptor<std::string> rpc_login; const command_line::arg_descriptor<bool> confirm_external_bind; const command_line::arg_descriptor<std::string> rpc_access_control_origins; @@ -76,6 +79,9 @@ namespace cryptonote static boost::optional<epee::net_utils::ssl_options_t> process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option = false); std::string bind_ip; + std::string bind_ipv6_address; + bool use_ipv6; + bool require_ipv4; std::vector<std::string> access_control_origins; boost::optional<tools::login> login; // currently `boost::none` if unspecified by user epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp index af5cb98a3..d528ffef3 100644 --- a/src/rpc/rpc_handler.cpp +++ b/src/rpc/rpc_handler.cpp @@ -63,7 +63,9 @@ namespace rpc d.cached_to -= 10; d.cached_top_hash = hash10; d.cached_m10_hash = crypto::null_hash; - d.cached_distribution.resize(d.cached_distribution.size() - 10); + CHECK_AND_ASSERT_MES(d.cached_distribution.size() >= 10, boost::none, "Cached distribution size does not match cached bounds"); + for (int p = 0; p < 10; ++p) + d.cached_distribution.pop_back(); can_extend = true; } } diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index ae748e052..668a2e5cd 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -59,7 +59,7 @@ void ZmqServer::serve() { throw std::runtime_error("ZMQ RPC server reply socket is null"); } - while (rep_socket->recv(&message)) + while (rep_socket->recv(&message, 0)) { std::string message_string(reinterpret_cast<const char *>(message.data()), message.size()); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index fca14c8f8..24a6eec43 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -201,11 +201,11 @@ namespace const char* USAGE_SET_DESCRIPTION("set_description [free text note]"); const char* USAGE_SIGN("sign <filename>"); const char* USAGE_VERIFY("verify <filename> <address> <signature>"); - const char* USAGE_EXPORT_KEY_IMAGES("export_key_images <filename>"); + const char* USAGE_EXPORT_KEY_IMAGES("export_key_images [all] <filename>"); const char* USAGE_IMPORT_KEY_IMAGES("import_key_images <filename>"); const char* USAGE_HW_KEY_IMAGES_SYNC("hw_key_images_sync"); const char* USAGE_HW_RECONNECT("hw_reconnect"); - const char* USAGE_EXPORT_OUTPUTS("export_outputs <filename>"); + const char* USAGE_EXPORT_OUTPUTS("export_outputs [all] <filename>"); const char* USAGE_IMPORT_OUTPUTS("import_outputs <filename>"); const char* USAGE_SHOW_TRANSFER("show_transfer <txid>"); const char* USAGE_MAKE_MULTISIG("make_multisig <threshold> <string1> [<string>...]"); @@ -803,6 +803,12 @@ bool simple_wallet::encrypted_seed(const std::vector<std::string> &args/* = std: return print_seed(true); } +bool simple_wallet::restore_height(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + success_msg_writer() << m_wallet->get_refresh_from_block_height(); + return true; +} + bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if (m_wallet->key_on_device()) @@ -2845,6 +2851,9 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), tr("Display the Electrum-style mnemonic seed")); + m_cmd_binder.set_handler("restore_height", + boost::bind(&simple_wallet::restore_height, this, _1), + tr("Display the restore height")); m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr(USAGE_SET_VARIABLE), @@ -2888,7 +2897,7 @@ simple_wallet::simple_wallet() "segregate-pre-fork-outputs <1|0>\n " " Set this if you intend to spend outputs on both Monero AND a key reusing fork.\n " "key-reuse-mitigation2 <1|0>\n " - " Set this if you are not sure whether you will spend on a key reusing Monero fork later.\n" + " Set this if you are not sure whether you will spend on a key reusing Monero fork later.\n " "subaddress-lookahead <major>:<minor>\n " " Set the lookahead sizes for the subaddress hash table.\n " " Set this if you are not sure whether you will spend on a key reusing Monero fork later.\n " @@ -3130,7 +3139,7 @@ simple_wallet::simple_wallet() tr("Available options:\n " "auto-send <1|0>\n " " Whether to automatically send newly generated messages right away.\n ")); - m_cmd_binder.set_handler("mms send_message_config", + m_cmd_binder.set_handler("mms send_signer_config", boost::bind(&simple_wallet::mms, this, _1), tr(USAGE_MMS_SEND_SIGNER_CONFIG), tr("Send completed signer config to all other authorized signers")); @@ -3972,7 +3981,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty()) { - m_wallet->explicit_refresh_from_block_height(!(command_line::is_arg_defaulted(vm, arg_restore_height) || + m_wallet->explicit_refresh_from_block_height(!(command_line::is_arg_defaulted(vm, arg_restore_height) && command_line::is_arg_defaulted(vm, arg_restore_date))); if (command_line::is_arg_defaulted(vm, arg_restore_height) && !command_line::is_arg_defaulted(vm, arg_restore_date)) { @@ -4089,7 +4098,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) m_wallet->callback(this); - check_background_mining(password); + bool skip_check_backround_mining = !command_line::get_arg(vm, arg_command).empty(); + if (!skip_check_backround_mining) + check_background_mining(password); if (welcome) message_writer(console_color_yellow, true) << tr("If you are new to Monero, type \"welcome\" for a brief overview."); @@ -7809,7 +7820,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec uint64_t fee = pd.m_amount_in - pd.m_amount_out; std::vector<std::pair<std::string, uint64_t>> destinations; for (const auto &d: pd.m_dests) { - destinations.push_back({get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr), d.amount}); + destinations.push_back({d.address(m_wallet->nettype(), pd.m_payment_id), d.amount}); } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) @@ -7885,7 +7896,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec uint64_t fee = amount - pd.m_amount_out; std::vector<std::pair<std::string, uint64_t>> destinations; for (const auto &d: pd.m_dests) { - destinations.push_back({get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr), d.amount}); + destinations.push_back({d.address(m_wallet->nettype(), pd.m_payment_id), d.amount}); } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) @@ -9042,21 +9053,31 @@ bool simple_wallet::verify(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::export_key_images(const std::vector<std::string> &args) +bool simple_wallet::export_key_images(const std::vector<std::string> &args_) { if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; } - if (args.size() != 1) + auto args = args_; + + if (m_wallet->watch_only()) { - PRINT_USAGE(USAGE_EXPORT_KEY_IMAGES); + fail_msg_writer() << tr("wallet is watch-only and cannot export key images"); return true; } - if (m_wallet->watch_only()) + + bool all = false; + if (args.size() >= 2 && args[0] == "all") { - fail_msg_writer() << tr("wallet is watch-only and cannot export key images"); + all = true; + args.erase(args.begin()); + } + + if (args.size() != 1) + { + PRINT_USAGE(USAGE_EXPORT_KEY_IMAGES); return true; } @@ -9068,7 +9089,7 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args) try { - if (!m_wallet->export_key_images(filename)) + if (!m_wallet->export_key_images(filename, all)) { fail_msg_writer() << tr("failed to save file ") << filename; return true; @@ -9193,13 +9214,22 @@ bool simple_wallet::hw_reconnect(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::export_outputs(const std::vector<std::string> &args) +bool simple_wallet::export_outputs(const std::vector<std::string> &args_) { if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + auto args = args_; + + bool all = false; + if (args.size() >= 2 && args[0] == "all") + { + all = true; + args.erase(args.begin()); + } + if (args.size() != 1) { PRINT_USAGE(USAGE_EXPORT_OUTPUTS); @@ -9214,7 +9244,7 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args) try { - std::string data = m_wallet->export_outputs_to_str(); + std::string data = m_wallet->export_outputs_to_str(all); bool r = epee::file_io_utils::save_string_to_file(filename, data); if (!r) { @@ -9341,7 +9371,7 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) for (const auto &d: pd.m_dests) { if (!dests.empty()) dests += ", "; - dests += get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) + ": " + print_money(d.amount); + dests += d.address(m_wallet->nettype(), pd.m_payment_id) + ": " + print_money(d.amount); } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 4bf7fa334..2de390666 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -109,6 +109,7 @@ namespace cryptonote bool spendkey(const std::vector<std::string> &args = std::vector<std::string>()); bool seed(const std::vector<std::string> &args = std::vector<std::string>()); bool encrypted_seed(const std::vector<std::string> &args = std::vector<std::string>()); + bool restore_height(const std::vector<std::string> &args = std::vector<std::string>()); /*! * \brief Sets seed language. diff --git a/src/wallet/api/transaction_history.cpp b/src/wallet/api/transaction_history.cpp index f4ad8b1f6..ad7029a3c 100644 --- a/src/wallet/api/transaction_history.cpp +++ b/src/wallet/api/transaction_history.cpp @@ -181,7 +181,7 @@ void TransactionHistoryImpl::refresh() // single output transaction might contain multiple transfers for (const auto &d: pd.m_dests) { - ti->m_transfers.push_back({d.amount, get_account_address_as_str(m_wallet->m_wallet->nettype(), d.is_subaddress, d.addr)}); + ti->m_transfers.push_back({d.amount, d.address(m_wallet->m_wallet->nettype(), pd.m_payment_id)}); } m_history.push_back(ti); } diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 03db385a4..e632b8d23 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -1407,8 +1407,7 @@ PendingTransaction* WalletImpl::restoreMultisigTransaction(const string& signDat // - unconfirmed_transfer_details; // - confirmed_transfer_details) -PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const string &payment_id, optional<uint64_t> amount, uint32_t mixin_count, - PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) +PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<string> &dst_addr, const string &payment_id, optional<std::vector<uint64_t>> amount, uint32_t mixin_count, PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { clearStatus(); @@ -1429,75 +1428,75 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const PendingTransactionImpl * transaction = new PendingTransactionImpl(*this); do { - if(!cryptonote::get_account_address_from_str(info, m_wallet->nettype(), dst_addr)) { - // TODO: copy-paste 'if treating as an address fails, try as url' from simplewallet.cpp:1982 - setStatusError(tr("Invalid destination address")); + std::vector<uint8_t> extra; + std::string extra_nonce; + vector<cryptonote::tx_destination_entry> dsts; + if (!amount && dst_addr.size() > 1) { + setStatusError(tr("Sending all requires one destination address")); break; } - - - std::vector<uint8_t> extra; - // if dst_addr is not an integrated address, parse payment_id - if (!info.has_payment_id && !payment_id.empty()) { - // copy-pasted from simplewallet.cpp:2212 + if (amount && (dst_addr.size() != (*amount).size())) { + setStatusError(tr("Destinations and amounts are unequal")); + break; + } + if (!payment_id.empty()) { crypto::hash payment_id_long; - bool r = tools::wallet2::parse_long_payment_id(payment_id, payment_id_long); - if (r) { - std::string extra_nonce; + if (tools::wallet2::parse_long_payment_id(payment_id, payment_id_long)) { cryptonote::set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id_long); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); } else { - r = tools::wallet2::parse_short_payment_id(payment_id, info.payment_id); - if (r) { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - } - } - - if (!r) { - setStatusError(tr("payment id has invalid format, expected 16 or 64 character hex string: ") + payment_id); + setStatusError(tr("payment id has invalid format, expected 64 character hex string: ") + payment_id); break; } } - else if (info.has_payment_id) { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); - bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - if (!r) { - setStatusError(tr("Failed to add short payment id: ") + epee::string_tools::pod_to_hex(info.payment_id)); + bool error = false; + for (size_t i = 0; i < dst_addr.size() && !error; i++) { + if(!cryptonote::get_account_address_from_str(info, m_wallet->nettype(), dst_addr[i])) { + // TODO: copy-paste 'if treating as an address fails, try as url' from simplewallet.cpp:1982 + setStatusError(tr("Invalid destination address")); + error = true; break; } - } - - - //std::vector<tools::wallet2::pending_tx> ptx_vector; + if (info.has_payment_id) { + if (!extra_nonce.empty()) { + setStatusError(tr("a single transaction cannot use more than one payment id")); + error = true; + break; + } + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + } - try { if (amount) { - vector<cryptonote::tx_destination_entry> dsts; cryptonote::tx_destination_entry de; - de.original = dst_addr; + de.original = dst_addr[i]; de.addr = info.address; - de.amount = *amount; + de.amount = (*amount)[i]; de.is_subaddress = info.is_subaddress; de.is_integrated = info.has_payment_id; dsts.push_back(de); - transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, - adjusted_priority, - extra, subaddr_account, subaddr_indices); } else { - // for the GUI, sweep_all (i.e. amount set as "(all)") will always sweep all the funds in all the addresses - if (subaddr_indices.empty()) - { + if (subaddr_indices.empty()) { for (uint32_t index = 0; index < m_wallet->get_num_subaddresses(subaddr_account); ++index) subaddr_indices.insert(index); } + } + } + if (error) { + break; + } + if (!extra_nonce.empty() && !add_extra_nonce_to_tx_extra(extra, extra_nonce)) { + setStatusError(tr("failed to set up payment id, though it was decoded correctly")); + break; + } + try { + if (amount) { + transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, + adjusted_priority, + extra, subaddr_account, subaddr_indices); + } else { transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, 1, fake_outs_count, 0 /* unlock_time */, - adjusted_priority, - extra, subaddr_account, subaddr_indices); + adjusted_priority, + extra, subaddr_account, subaddr_indices); } - pendingTxPostProcess(transaction); if (multisig().isMultisig) { @@ -1574,6 +1573,13 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const return transaction; } +PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const string &payment_id, optional<uint64_t> amount, uint32_t mixin_count, + PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) + +{ + return createTransactionMultDest(std::vector<string> {dst_addr}, payment_id, amount ? (std::vector<uint64_t> {*amount}) : (optional<std::vector<uint64_t>>()), mixin_count, priority, subaddr_account, subaddr_indices); +} + PendingTransaction *WalletImpl::createSweepUnmixableTransaction() { @@ -1705,7 +1711,9 @@ bool WalletImpl::setCacheAttribute(const std::string &key, const std::string &va std::string WalletImpl::getCacheAttribute(const std::string &key) const { - return m_wallet->get_attribute(key); + std::string value; + m_wallet->get_attribute(key, value); + return value; } bool WalletImpl::setUserNote(const std::string &txid, const std::string ¬e) @@ -1740,18 +1748,27 @@ std::string WalletImpl::getTxKey(const std::string &txid_str) const crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; - if (m_wallet->get_tx_key(txid, tx_key, additional_tx_keys)) + try { clearStatus(); - std::ostringstream oss; - oss << epee::string_tools::pod_to_hex(tx_key); - for (size_t i = 0; i < additional_tx_keys.size(); ++i) - oss << epee::string_tools::pod_to_hex(additional_tx_keys[i]); - return oss.str(); + if (m_wallet->get_tx_key(txid, tx_key, additional_tx_keys)) + { + clearStatus(); + std::ostringstream oss; + oss << epee::string_tools::pod_to_hex(tx_key); + for (size_t i = 0; i < additional_tx_keys.size(); ++i) + oss << epee::string_tools::pod_to_hex(additional_tx_keys[i]); + return oss.str(); + } + else + { + setStatusError(tr("no tx keys found for this txid")); + return ""; + } } - else + catch (const std::exception &e) { - setStatusError(tr("no tx keys found for this txid")); + setStatusError(e.what()); return ""; } } diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index a367a1917..331bf4b38 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -149,6 +149,11 @@ public: bool hasMultisigPartialKeyImages() const override; PendingTransaction* restoreMultisigTransaction(const std::string& signData) override; + PendingTransaction * createTransactionMultDest(const std::vector<std::string> &dst_addr, const std::string &payment_id, + optional<std::vector<uint64_t>> amount, uint32_t mixin_count, + PendingTransaction::Priority priority = PendingTransaction::Priority_Low, + uint32_t subaddr_account = 0, + std::set<uint32_t> subaddr_indices = {}) override; PendingTransaction * createTransaction(const std::string &dst_addr, const std::string &payment_id, optional<uint64_t> amount, uint32_t mixin_count, PendingTransaction::Priority priority = PendingTransaction::Priority_Low, diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 9e556cb2f..e543a115b 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -812,6 +812,26 @@ struct Wallet * @return PendingTransaction */ virtual PendingTransaction* restoreMultisigTransaction(const std::string& signData) = 0; + + /*! + * \brief createTransactionMultDest creates transaction with multiple destinations. if dst_addr is an integrated address, payment_id is ignored + * \param dst_addr vector of destination address as string + * \param payment_id optional payment_id, can be empty string + * \param amount vector of amounts + * \param mixin_count mixin count. if 0 passed, wallet will use default value + * \param subaddr_account subaddress account from which the input funds are taken + * \param subaddr_indices set of subaddress indices to use for transfer or sweeping. if set empty, all are chosen when sweeping, and one or more are automatically chosen when transferring. after execution, returns the set of actually used indices + * \param priority + * \return PendingTransaction object. caller is responsible to check PendingTransaction::status() + * after object returned + */ + + virtual PendingTransaction * createTransactionMultDest(const std::vector<std::string> &dst_addr, const std::string &payment_id, + optional<std::vector<uint64_t>> amount, uint32_t mixin_count, + PendingTransaction::Priority = PendingTransaction::Priority_Low, + uint32_t subaddr_account = 0, + std::set<uint32_t> subaddr_indices = {}) = 0; + /*! * \brief createTransaction creates transaction. if dst_addr is an integrated address, payment_id is ignored * \param dst_addr destination address as string diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f25e9ad97..9782e4b1e 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -217,6 +217,8 @@ namespace add_reason(reason, "invalid input"); if (res.invalid_output) add_reason(reason, "invalid output"); + if (res.too_few_outputs) + add_reason(reason, "too few outputs"); if (res.too_big) add_reason(reason, "too big"); if (res.overspend) @@ -6123,7 +6125,7 @@ void wallet2::commit_tx(pending_tx& ptx) amount_in += m_transfers[idx].amount(); } add_unconfirmed_tx(ptx.tx, amount_in, dests, payment_id, ptx.change_dts.amount, ptx.construction_data.subaddr_account, ptx.construction_data.subaddr_indices); - if (store_tx_info()) + if (store_tx_info() && ptx.tx_key != crypto::null_skey) { m_tx_keys.insert(std::make_pair(txid, ptx.tx_key)); m_additional_tx_keys.insert(std::make_pair(txid, ptx.additional_tx_keys)); @@ -6312,7 +6314,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin // normally, the tx keys are saved in commit_tx, when the tx is actually sent to the daemon. // we can't do that here since the tx will be sent from the compromised wallet, which we don't want // to see that info, so we save it here - if (store_tx_info()) + if (store_tx_info() && ptx.tx_key != crypto::null_skey) { const crypto::hash txid = get_transaction_hash(ptx.tx); m_tx_keys.insert(std::make_pair(txid, tx_key)); @@ -7448,7 +7450,7 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ order.resize(light_wallet_requested_outputs_count); for (size_t n = 0; n < order.size(); ++n) order[n] = n; - std::shuffle(order.begin(), order.end(), std::default_random_engine(crypto::rand<unsigned>())); + std::shuffle(order.begin(), order.end(), crypto::random_device{}); LOG_PRINT_L2("Looking for " << (fake_outputs_count+1) << " outputs with amounts " << print_money(td.is_rct() ? 0 : td.amount())); @@ -8023,7 +8025,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> order.resize(requested_outputs_count); for (size_t n = 0; n < order.size(); ++n) order[n] = n; - std::shuffle(order.begin(), order.end(), std::default_random_engine(crypto::rand<unsigned>())); + std::shuffle(order.begin(), order.end(), crypto::random_device{}); LOG_PRINT_L2("Looking for " << (fake_outputs_count+1) << " outputs of size " << print_money(td.is_rct() ? 0 : td.amount())); for (size_t o = 0; o < requested_outputs_count && outs.back().size() < fake_outputs_count + 1; ++o) @@ -8165,6 +8167,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent if (needed_money < found_money) { change_dts.addr = get_subaddress({subaddr_account, 0}); + change_dts.is_subaddress = subaddr_account != 0; change_dts.amount = found_money - needed_money; } @@ -10308,6 +10311,8 @@ bool wallet2::get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx if (i == m_tx_keys.end()) return false; tx_key = i->second; + if (tx_key == crypto::null_skey) + return false; const auto j = m_additional_tx_keys.find(txid); if (j != m_additional_tx_keys.end()) additional_tx_keys = j->second; @@ -10319,6 +10324,7 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s bool r = get_tx_key_cached(txid, tx_key, additional_tx_keys); if (r) { + MDEBUG("tx key cached for txid: " << txid); return true; } @@ -10380,13 +10386,18 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s dev_cold->get_tx_key(tx_keys, tx_key_data, m_account.get_keys().m_view_secret_key); if (tx_keys.empty()) { + MDEBUG("Empty tx keys for txid: " << txid); + return false; + } + + if (tx_keys[0] == crypto::null_skey) + { return false; } tx_key = tx_keys[0]; tx_keys.erase(tx_keys.begin()); additional_tx_keys = tx_keys; - return true; } //---------------------------------------------------------------------------------------------------- @@ -11596,10 +11607,10 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle return tx_pub_key; } -bool wallet2::export_key_images(const std::string &filename) const +bool wallet2::export_key_images(const std::string &filename, bool all) const { PERF_TIMER(export_key_images); - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const uint32_t offset = ski.first; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index a6d042297..c2e34dd76 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1175,7 +1175,7 @@ private: void import_payments_out(const std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> &confirmed_payments); std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); - bool export_key_images(const std::string &filename) const; + bool export_key_images(const std::string &filename, bool all = false) const; std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index a4bb342ca..9da9d109c 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -206,7 +206,10 @@ namespace wallet_args if (!command_line::is_arg_defaulted(vm, arg_log_level)) MINFO("Setting log level = " << command_line::get_arg(vm, arg_log_level)); else - MINFO("Setting log levels = " << getenv("MONERO_LOGS")); + { + const char *logs = getenv("MONERO_LOGS"); + MINFO("Setting log levels = " << (logs ? logs : "<default>")); + } MINFO(wallet_args::tr("Logging to: ") << log_path); Print(print) << boost::format(wallet_args::tr("Logging to %s")) % log_path; diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 9d3605d11..c64b662f3 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -247,7 +247,9 @@ namespace tools m_net_server.set_threads_prefix("RPC"); auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( - rng, std::move(bind_port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), + rng, std::move(bind_port), std::move(rpc_config->bind_ip), + std::move(rpc_config->bind_ipv6_address), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4), + std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); } @@ -350,7 +352,7 @@ namespace tools entry.destinations.push_back(wallet_rpc::transfer_destination()); wallet_rpc::transfer_destination &td = entry.destinations.back(); td.amount = d.amount; - td.address = d.original.empty() ? get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) : d.original; + td.address = d.address(m_wallet->nettype(), pd.m_payment_id); } entry.type = "out"; @@ -380,7 +382,7 @@ namespace tools entry.destinations.push_back(wallet_rpc::transfer_destination()); wallet_rpc::transfer_destination &td = entry.destinations.back(); td.amount = d.amount; - td.address = d.original.empty() ? get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) : d.original; + td.address = d.address(m_wallet->nettype(), pd.m_payment_id); } entry.type = is_failed ? "failed" : "pending"; @@ -4089,9 +4091,8 @@ namespace tools } } - er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; - er.message = std::string("Invalid address"); - return false; + res.valid = false; + return true; } //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_set_daemon(const wallet_rpc::COMMAND_RPC_SET_DAEMON::request& req, wallet_rpc::COMMAND_RPC_SET_DAEMON::response& res, epee::json_rpc::error& er, const connection_context *ctx) diff --git a/tests/functional_tests/blockchain.py b/tests/functional_tests/blockchain.py index 2bd7672c0..2c3f34c35 100755 --- a/tests/functional_tests/blockchain.py +++ b/tests/functional_tests/blockchain.py @@ -78,6 +78,12 @@ class BlockchainTest(): except: ok = True assert ok + res = daemon.get_fee_estimate() + assert res.fee == 234562 + assert res.quantization_mask == 10000 + res = daemon.get_fee_estimate(10) + assert res.fee <= 234562 + # generate blocks res_generateblocks = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', blocks) @@ -229,6 +235,12 @@ class BlockchainTest(): assert res.histogram[i].unlocked_instances == 0 assert res.histogram[i].recent_instances == 0 + res = daemon.get_fee_estimate() + assert res.fee == 234560 + assert res.quantization_mask == 10000 + res = daemon.get_fee_estimate(10) + assert res.fee <= 234560 + def _test_alt_chains(self): print('Testing alt chains') daemon = Daemon() diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 25ab641ab..77d0e4c4d 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -65,7 +65,7 @@ try: for i in range(len(command_lines)): #print('Running: ' + str(command_lines[i])) processes.append(subprocess.Popen(command_lines[i], stdout = outputs[i])) -except Exception, e: +except Exception as e: print('Error: ' + str(e)) sys.exit(1) diff --git a/tests/functional_tests/multisig.py b/tests/functional_tests/multisig.py index 3c8cd9c1d..b109acf91 100755 --- a/tests/functional_tests/multisig.py +++ b/tests/functional_tests/multisig.py @@ -43,6 +43,7 @@ class MultisigTest(): self.mine('42jSRGmmKN96V2j3B8X2DbiNThBXW1tSi1rW1uwkqbyURenq3eC3yosNm8HEMdHuWwKMFGzMUB3RCTvcTaW9kHpdRPP7p5y', 5) self.mine('47fF32AdrmXG84FcPY697uZdd42pMMGiH5UpiTRTt3YX2pZC7t7wkzEMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUgxRd53', 5) self.mine('44SKxxLQw929wRF6BA9paQ1EWFshNnKhXM3qz6Mo3JGDE2YG3xyzVutMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUiqhcwR', 5) + self.mine('4ADHswEU3XBUee8pudBkZQd9beJainqNo1BQKkHJujAEPJyQrLj9U4dNm8HEMdHuWwKMFGzMUB3RCTvcTaW9kHpdRUDxgjW', 5) self.mine('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 60) self.create_multisig_wallets(2, 2, '493DsrfJPqiN3Suv9RcRDoZEbQtKZX1sNcGPA3GhkKYEEmivk8kjQrTdRdVc4ZbmzWJuE157z9NNUKmF2VDfdYDR3CziGMk') @@ -57,6 +58,12 @@ class MultisigTest(): self.import_multisig_info([0, 1, 2], 6) self.check_transaction(txid) + self.create_multisig_wallets(3, 3, '4ADHswEU3XBUee8pudBkZQd9beJainqNo1BQKkHJujAEPJyQrLj9U4dNm8HEMdHuWwKMFGzMUB3RCTvcTaW9kHpdRUDxgjW') + self.import_multisig_info([2, 0, 1], 5) + txid = self.transfer([2, 1, 0]) + self.import_multisig_info([0, 2, 1], 6) + self.check_transaction(txid) + self.create_multisig_wallets(3, 4, '47fF32AdrmXG84FcPY697uZdd42pMMGiH5UpiTRTt3YX2pZC7t7wkzEMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUgxRd53') self.import_multisig_info([0, 2, 3], 5) txid = self.transfer([0, 2, 3]) diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py index d9a6e592e..7ebda6ebd 100755 --- a/tests/functional_tests/transfer.py +++ b/tests/functional_tests/transfer.py @@ -46,6 +46,7 @@ class TransferTest(): self.check_get_bulk_payments() self.check_double_spend_detection() self.sweep_single() + self.check_destinations() def reset(self): print('Resetting blockchain') @@ -574,6 +575,7 @@ class TransferTest(): assert res.overspend == False assert res.fee_too_low == False assert res.not_rct == False + assert res.too_few_outputs == False res = daemon.get_transactions([txes[0][0]]) assert len(res.txs) >= 1 @@ -629,6 +631,64 @@ class TransferTest(): res = self.wallet[0].incoming_transfers(transfer_type = 'unavailable') assert len([t for t in res.transfers if t.key_image == ki]) == 1 + def check_destinations(self): + daemon = Daemon() + + print("Checking transaction destinations") + + dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000} + res = self.wallet[0].transfer([dst]) + assert len(res.tx_hash) == 64 + tx_hash = res.tx_hash + for i in range(2): + res = self.wallet[0].get_transfers(pending = True, out = True) + l = [x for x in (res.pending if i == 0 else res.out) if x.txid == tx_hash] + assert len(l) == 1 + e = l[0] + assert len(e.destinations) == 1 + assert e.destinations[0].amount == 1000000000000 + assert e.destinations[0].address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + + if i == 0: + daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1) + self.wallet[0].refresh() + + dst = {'address': '8AsN91rznfkBGTY8psSNkJBg9SZgxxGGRUhGwRptBhgr5XSQ1XzmA9m8QAnoxydecSh5aLJXdrgXwTDMMZ1AuXsN1EX5Mtm', 'amount': 1000000000000} + res = self.wallet[0].transfer([dst]) + assert len(res.tx_hash) == 64 + tx_hash = res.tx_hash + for i in range(2): + res = self.wallet[0].get_transfers(pending = True, out = True) + l = [x for x in (res.pending if i == 0 else res.out) if x.txid == tx_hash] + assert len(l) == 1 + e = l[0] + assert len(e.destinations) == 1 + assert e.destinations[0].amount == 1000000000000 + assert e.destinations[0].address == '8AsN91rznfkBGTY8psSNkJBg9SZgxxGGRUhGwRptBhgr5XSQ1XzmA9m8QAnoxydecSh5aLJXdrgXwTDMMZ1AuXsN1EX5Mtm' + + if i == 0: + daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1) + self.wallet[0].refresh() + + dst = {'address': '4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY', 'amount': 1000000000000} + res = self.wallet[0].transfer([dst]) + assert len(res.tx_hash) == 64 + tx_hash = res.tx_hash + for i in range(2): + res = self.wallet[0].get_transfers(pending = True, out = True) + l = [x for x in (res.pending if i == 0 else res.out) if x.txid == tx_hash] + assert len(l) == 1 + e = l[0] + assert len(e.destinations) == 1 + assert e.destinations[0].amount == 1000000000000 + assert e.destinations[0].address == '4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY' + + if i == 0: + daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1) + self.wallet[0].refresh() + + + if __name__ == '__main__': TransferTest().run_test() diff --git a/tests/functional_tests/validate_address.py b/tests/functional_tests/validate_address.py new file mode 100755 index 000000000..58748b0a2 --- /dev/null +++ b/tests/functional_tests/validate_address.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2019 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import time + +"""Test address validation RPC calls +""" + +from framework.wallet import Wallet + +class AddressValidationTest(): + def run_test(self): + self.create() + self.check_bad_addresses() + self.check_good_addresses() + self.check_openalias_addresses() + + def create(self): + print('Creating wallet') + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + self.wallet = Wallet() + # close the wallet if any, will throw if none is loaded + try: self.wallet.close_wallet() + except: pass + res = self.wallet.restore_deterministic_wallet(seed = seed) + assert res.address == address + assert res.seed == seed + + def check_bad_addresses(self): + print('Validating bad addresses') + bad_addresses = ['', 'a', '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWD9', ' ', '@', '42ey'] + for address in bad_addresses: + res = self.wallet.validate_address(address, any_net_type = False) + assert not res.valid + res = self.wallet.validate_address(address, any_net_type = True) + assert not res.valid + + def check_good_addresses(self): + print('Validating good addresses') + addresses = [ + [ 'mainnet', '', '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' ], + [ 'mainnet', '', '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW' ], + [ 'testnet', '', '9ujeXrjzf7bfeK3KZdCqnYaMwZVFuXemPU8Ubw335rj2FN1CdMiWNyFV3ksEfMFvRp9L9qum5UxkP5rN9aLcPxbH1au4WAB' ], + [ 'stagenet', '', '53teqCAESLxeJ1REzGMAat1ZeHvuajvDiXqboEocPaDRRmqWoVPzy46GLo866qRFjbNhfkNckyhST3WEvBviDwpUDd7DSzB' ], + [ 'mainnet', 'i', '4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY' ], + [ 'mainnet', 's', '8AsN91rznfkBGTY8psSNkJBg9SZgxxGGRUhGwRptBhgr5XSQ1XzmA9m8QAnoxydecSh5aLJXdrgXwTDMMZ1AuXsN1EX5Mtm' ], + [ 'mainnet', 's', '86kKnBKFqzCLxtK1Jmx2BkNBDBSMDEVaRYMMyVbeURYDWs8uNGDZURKCA5yRcyMxHzPcmCf1q2fSdhQVcaKsFrtGRsdGfNk' ], + [ 'testnet', 'i', 'AApMA1VuhiCaHzr5X2KXi2Zc9oJ3VaGjkfChxxpRpxkyKf1NetvbRbQTbFMrGkr85DjnEH7JsBaoUFsgKwZnmtnVWnoB8MDotCsLb7eWwz' ], + [ 'testnet', 's', 'BdKg9udkvckC5T58a8Nmtb6BNsgRAxs7uA2D49sWNNX5HPW5Us6Wxu8QMXrnSx3xPBQQ2iu9kwEcRGAoiz6EPmcZKbF62GS' ], + [ 'testnet', 's', 'BcFvPa3fT4gVt5QyRDe5Vv7VtUFao9ci8NFEy3r254KF7R1N2cNB5FYhGvrHbMStv4D6VDzZ5xtxeKV8vgEPMnDcNFuwZb9' ], + [ 'stagenet', 'i', '5K8mwfjumVseCcQEjNbf59Um6R9NfVUNkHTLhhPCmNvgDLVS88YW5tScnm83rw9mfgYtchtDDTW5jEfMhygi27j1QYphX38hg6m4VMtN29' ], + [ 'stagenet', 's', '73LhUiix4DVFMcKhsPRG51QmCsv8dYYbL6GcQoLwEEFvPvkVvc7BhebfA4pnEFF9Lq66hwvLqBvpHjTcqvpJMHmmNjPPBqa' ], + [ 'stagenet', 's', '7A1Hr63MfgUa8pkWxueD5xBqhQczkusYiCMYMnJGcGmuQxa7aDBxN1G7iCuLCNB3VPeb2TW7U9FdxB27xKkWKfJ8VhUZthF' ], + ] + for any_net_type in [True, False]: + for address in addresses: + res = self.wallet.validate_address(address[2], any_net_type = any_net_type) + if any_net_type or address[0] == 'mainnet': + assert res.valid + assert res.integrated == (address[1] == 'i') + assert res.subaddress == (address[1] == 's') + assert res.nettype == address[0] + assert res.openalias_address == '' + else: + assert not res.valid + + def check_openalias_addresses(self): + print('Validating openalias addresses') + addresses = [ + ['donate@getmonero.org', '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A'] + ] + for address in addresses: + res = self.wallet.validate_address(address[0]) + assert not res.valid + res = self.wallet.validate_address(address[0], allow_openalias = True) + assert res.valid + assert not res.integrated + assert not res.subaddress + assert res.nettype == 'mainnet' + assert res.openalias_address == address[1] + +if __name__ == '__main__': + AddressValidationTest().run_test() diff --git a/tests/functional_tests/wallet_address.py b/tests/functional_tests/wallet_address.py index 4ff059a6f..eda52b432 100755 --- a/tests/functional_tests/wallet_address.py +++ b/tests/functional_tests/wallet_address.py @@ -198,8 +198,9 @@ class WalletAddressTest(): try: wallet.close_wallet() except: pass languages = res.languages - for language in languages: - print('Creating ' + str(language) + ' wallet') + languages_local = res.languages_local + for language in languages + languages_local: + print('Creating ' + language.encode('utf8') + ' wallet') wallet.create_wallet(filename = '', language = language) res = wallet.query_key('mnemonic') wallet.close_wallet() diff --git a/tests/unit_tests/blockchain_db.cpp b/tests/unit_tests/blockchain_db.cpp index f302d7946..d7c60cecb 100644 --- a/tests/unit_tests/blockchain_db.cpp +++ b/tests/unit_tests/blockchain_db.cpp @@ -162,7 +162,7 @@ protected: { block bl; blobdata bd = h2b(i); - parse_and_validate_block_from_blob(bd, bl); + CHECK_AND_ASSERT_THROW_MES(parse_and_validate_block_from_blob(bd, bl), "Invalid block"); m_blocks.push_back(std::make_pair(bl, bd)); } for (auto& i : t_transactions) @@ -172,7 +172,7 @@ protected: { transaction tx; blobdata bd = h2b(j); - parse_and_validate_tx_from_blob(bd, tx); + CHECK_AND_ASSERT_THROW_MES(parse_and_validate_tx_from_blob(bd, tx), "Invalid transaction"); txs.push_back(std::make_pair(tx, bd)); } m_txs.push_back(txs); diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 946731826..32328edd9 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <array> +#include <boost/predef/other/endian.h> #include <boost/endian/conversion.hpp> #include <boost/range/algorithm/equal.hpp> #include <boost/range/algorithm_ext/iota.hpp> @@ -135,7 +136,7 @@ namespace EXPECT_FALSE( lhs >= rhs ); \ EXPECT_TRUE( rhs >= lhs ) - #ifdef BOOST_LITTLE_ENDIAN + #if BOOST_ENDIAN_LITTLE_BYTE #define CHECK_LESS_ENDIAN(lhs, rhs) CHECK_LESS( rhs , lhs ) #else #define CHECK_LESS_ENDIAN(lhs, rhs) CHECK_LESS( lhs , rhs ) @@ -946,3 +947,20 @@ TEST(parsing, number) epee::misc_utils::parse::match_number(i, s.end(), val); ASSERT_EQ(val, "+9.34e+03"); } + +TEST(parsing, unicode) +{ + std::string bs; + std::string s; + std::string::const_iterator si; + + s = "\"\""; si = s.begin(); ASSERT_TRUE(epee::misc_utils::parse::match_string(si, s.end(), bs)); ASSERT_EQ(bs, ""); + s = "\"\\u0000\""; si = s.begin(); ASSERT_TRUE(epee::misc_utils::parse::match_string(si, s.end(), bs)); ASSERT_EQ(bs, std::string(1, '\0')); + s = "\"\\u0020\""; si = s.begin(); ASSERT_TRUE(epee::misc_utils::parse::match_string(si, s.end(), bs)); ASSERT_EQ(bs, " "); + s = "\"\\u1\""; si = s.begin(); ASSERT_FALSE(epee::misc_utils::parse::match_string(si, s.end(), bs)); + s = "\"\\u12\""; si = s.begin(); ASSERT_FALSE(epee::misc_utils::parse::match_string(si, s.end(), bs)); + s = "\"\\u123\""; si = s.begin(); ASSERT_FALSE(epee::misc_utils::parse::match_string(si, s.end(), bs)); + s = "\"\\u1234\""; si = s.begin(); ASSERT_TRUE(epee::misc_utils::parse::match_string(si, s.end(), bs)); ASSERT_EQ(bs, "ሴ"); + s = "\"foo\\u1234bar\""; si = s.begin(); ASSERT_TRUE(epee::misc_utils::parse::match_string(si, s.end(), bs)); ASSERT_EQ(bs, "fooሴbar"); + s = "\"\\u3042\\u307e\\u3084\\u304b\\u3059\""; si = s.begin(); ASSERT_TRUE(epee::misc_utils::parse::match_string(si, s.end(), bs)); ASSERT_EQ(bs, "あまやかす"); +} diff --git a/tests/unit_tests/output_selection.cpp b/tests/unit_tests/output_selection.cpp index c98696fbd..0724cd3e0 100644 --- a/tests/unit_tests/output_selection.cpp +++ b/tests/unit_tests/output_selection.cpp @@ -211,10 +211,10 @@ TEST(select_outputs, same_distribution) { const double diff = (double)output_norm[i] - (double)chain_norm[i]; double dev = fabs(2.0 * diff / (output_norm[i] + chain_norm[i])); - ASSERT_LT(dev, 0.1); + ASSERT_LT(dev, 0.15); avg_dev += dev; } avg_dev /= 100; MDEBUG("avg_dev: " << avg_dev); - ASSERT_LT(avg_dev, 0.015); + ASSERT_LT(avg_dev, 0.02); } diff --git a/tests/unit_tests/ringct.cpp b/tests/unit_tests/ringct.cpp index 4d51ec434..d2b2c3109 100644 --- a/tests/unit_tests/ringct.cpp +++ b/tests/unit_tests/ringct.cpp @@ -779,8 +779,8 @@ TEST(ringct, range_proofs_accept_very_long_simple) inputs[n] = n; outputs[n] = n; } - std::random_shuffle(inputs, inputs + N); - std::random_shuffle(outputs, outputs + N); + std::shuffle(inputs, inputs + N, crypto::random_device{}); + std::shuffle(outputs, outputs + N, crypto::random_device{}); EXPECT_TRUE(range_proof_test(true, NELTS(inputs), inputs, NELTS(outputs), outputs, false, true)); } diff --git a/tests/unit_tests/rolling_median.cpp b/tests/unit_tests/rolling_median.cpp index 547fe092f..9e4cf87b8 100644 --- a/tests/unit_tests/rolling_median.cpp +++ b/tests/unit_tests/rolling_median.cpp @@ -143,7 +143,7 @@ TEST(rolling_median, order) m.insert(random[i]); ASSERT_EQ(med, m.median()); - std::shuffle(random.begin(), random.end(), std::default_random_engine(crypto::rand<unsigned>())); + std::shuffle(random.begin(), random.end(), crypto::random_device{}); m.clear(); for (int i = 0; i < 1000; ++i) m.insert(random[i]); diff --git a/translations/ready b/translations/ready new file mode 100644 index 000000000..a6f2fd872 --- /dev/null +++ b/translations/ready @@ -0,0 +1,4 @@ +fr +it +ja +sv diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 18ce37221..23d5ec0f0 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -90,11 +90,12 @@ class Daemon(object): } return self.rpc.send_json_rpc_request(getlastblockheader) - def getblockheaderbyhash(self, hash): + def getblockheaderbyhash(self, hash = "", hashes = []): getblockheaderbyhash = { 'method': 'getblockheaderbyhash', 'params': { 'hash': hash, + 'hashes': hashes, }, 'jsonrpc': '2.0', 'id': '0' @@ -332,3 +333,14 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_alternate_chains) + + def get_fee_estimate(self, grace_blocks = 0): + get_fee_estimate = { + 'method': 'get_fee_estimate', + 'params': { + 'grace_blocks': grace_blocks, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_fee_estimate) diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index 2e2650f92..36ff3644f 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -750,6 +750,19 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(set_log_categories) + def validate_address(self, address, any_net_type = False, allow_openalias = False): + validate_address = { + 'method': 'validate_address', + 'params': { + 'address': address, + 'any_net_type': any_net_type, + 'allow_openalias': allow_openalias, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(validate_address) + def get_version(self): get_version = { 'method': 'get_version', diff --git a/utils/translations/build-translations.sh b/utils/translations/build-translations.sh index 0956e633f..1217dca0a 100755 --- a/utils/translations/build-translations.sh +++ b/utils/translations/build-translations.sh @@ -12,5 +12,15 @@ then fi echo "using $lrelease" -"$lrelease" translations/*.ts +if test -f translations/ready +then + languages="" + for language in $(cat translations/ready) + do + languages="$languages translations/$language.ts" + done +else + languages="translations/*.ts" +fi +"$lrelease" $languages |