diff options
84 files changed, 1510 insertions, 1023 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aa6afbfbc..5c52eb9c6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,15 @@ name: ci/gh-actions/cli on: [push, pull_request] +# The below variables reduce repetitions across similar targets +env: + REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost + APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache' + APT_SET_CONF: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + jobs: build-macos: runs-on: macOS-latest @@ -66,16 +75,13 @@ jobs: key: ccache-ubuntu-build-${{ github.sha }} restore-keys: ccache-ubuntu-build- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: build run: | ccache --max-size=150M @@ -96,16 +102,13 @@ jobs: key: ccache-ubuntu-libwallet-${{ github.sha }} restore-keys: ccache-ubuntu-libwallet- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: build run: | ccache --max-size=150M @@ -129,16 +132,13 @@ jobs: key: test-ubuntu-ccache-${{ github.sha }} restore-keys: test-ubuntu-ccache- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: install Python dependencies run: pip install requests psutil monotonic - name: tests diff --git a/CMakeLists.txt b/CMakeLists.txt index a39e2ed44..511c62a4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -525,6 +525,20 @@ add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP) set(MONERO_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_include") include_directories(${MONERO_GENERATED_HEADERS_DIR}) +option(COVERAGE "Enable profiling for test coverage report" OFF) +if(COVERAGE) + message(STATUS "Building with profiling for test coverage report") +endif() +macro (monero_enable_coverage) + if(COVERAGE) + foreach(COV_FLAG -fprofile-arcs -ftest-coverage --coverage) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COV_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COV_FLAG}") + endforeach() + endif() +endmacro() + + # Generate header for embedded translations # Generate header for embedded translations, use target toolchain if depends, otherwise use the # lrelease and lupdate binaries from the host @@ -688,13 +702,7 @@ else() set(STATIC_ASSERT_CPP_FLAG "-Dstatic_assert=_Static_assert") endif() - option(COVERAGE "Enable profiling for test coverage report" 0) - - if(COVERAGE) - message(STATUS "Building with profiling for test coverage report") - set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage --coverage") - endif() - + monero_enable_coverage() # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that # is fixed in the code (Issue #847), force compiler to be conservative. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") @@ -771,8 +779,8 @@ else() message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}") # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that @@ -92,7 +92,7 @@ As with many development projects, the repository on Github is considered to be Monero is a 100% community-sponsored endeavor. If you want to join our efforts, the easiest thing you can do is support the project financially. Both Monero and Bitcoin donations can be made to **donate.getmonero.org** if using a client that supports the [OpenAlias](https://openalias.org) standard. Alternatively, you can send XMR to the Monero donation address via the `donate` command (type `help` in the command-line wallet for details). -The Monero donation address is: `888tNkZrPN6JsEgekjMnABU4TBzc2Dt29EPAvkRxbANsAnjyPbb3iQ1YBRk1UXcdRsiKc9dhwMVgN5S9cQUiyoogDavup3H` (viewkey: `f359631075708155cc3d92a32b75a7d02a5dcf27756707b47a2b31b21c389501`) +The Monero donation address is: `888tNkZrPN6JsEgekjMnABU4TBzc2Dt29EPAvkRxbANsAnjyPbb3iQ1YBRk1UXcdRsiKc9dhwMVgN5S9cQUiyoogDavup3H` (viewkey: `f359631075708155cc3d92a32b75a7d02a5dcf27756707b47a2b31b21c389501`; base address for restoring with address and viewkey: `44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A`) The Bitcoin donation address is: `1KTexdemPdxSBcG55heUuTjDRYqbC5ZL8H` diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 511f0416e..046115bd3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -26,5 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +monero_enable_coverage() add_subdirectory(epee) diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 25f8c648b..84dc79266 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -24,211 +24,23 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include <fstream> -#include <boost/filesystem/path.hpp> -#include <boost/filesystem/operations.hpp> -#ifdef WIN32 -#include <windows.h> -#include "string_tools.h" -#endif - -// On Windows there is a problem with non-ASCII characters in path and file names -// as far as support by the standard components used is concerned: - -// The various file stream classes, e.g. std::ifstream and std::ofstream, are -// part of the GNU C++ Library / libstdc++. On the most basic level they use the -// fopen() call as defined / made accessible to programs compiled within MSYS2 -// by the stdio.h header file maintained by the MinGW project. - -// The critical point: The implementation of fopen() is part of MSVCRT, the -// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any -// Unicode support. - -// Monero code that would want to continue to use the normal file stream classes -// but WITH Unicode support could therefore not solve this problem on its own, -// but 2 different projects from 2 different maintaining groups would need changes -// in this particular direction - something probably difficult to achieve and -// with a long time to wait until all new versions / releases arrive. - -// Implemented solution approach: Circumvent the problem by stopping to use std -// file stream classes on Windows and directly use Unicode-capable WIN32 API -// calls. Most of the code doing so is concentrated in this header file here. +#include <string> +#include <ctime> namespace epee { namespace file_io_utils { - inline - bool is_file_exist(const std::string& path) - { - boost::filesystem::path p(path); - return boost::filesystem::exists(p); - } - - inline - bool save_string_to_file(const std::string& path_to_file, const std::string& str) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD bytes_written; - DWORD bytes_to_write = (DWORD)str.size(); - BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); - CloseHandle(file_handle); - if (bytes_written != bytes_to_write) - result = FALSE; - return result; -#else - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool get_file_time(const std::string& path_to_file, time_t& ft) - { - boost::system::error_code ec; - ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); - if(!ec) - return true; - else - return false; - } - - inline - bool set_file_time(const std::string& path_to_file, const time_t& ft) - { - boost::system::error_code ec; - boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); - if(!ec) - return true; - else - return false; - } - - - inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { - CloseHandle(file_handle); - return false; - } - target_str.resize(file_size); - DWORD bytes_read; - BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); - CloseHandle(file_handle); - if (bytes_read != file_size) - result = FALSE; - return result; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - - std::ifstream::pos_type file_size = fstream.tellg(); - - if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large - return false;//don't go crazy - size_t file_size_t = static_cast<size_t>(file_size); - - target_str.resize(file_size_t); - - fstream.seekg (0, std::ios::beg); - fstream.read((char*)target_str.data(), target_str.size()); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool append_string_to_file(const std::string& path_to_file, const std::string& str) - { - // No special Windows implementation because so far not used in Monero code - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } - } - - inline - bool get_file_size(const std::string& path_to_file, uint64_t &size) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER file_size; - BOOL result = GetFileSizeEx(file_handle, &file_size); - CloseHandle(file_handle); - if (result) { - size = file_size.QuadPart; - } - return size; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - size = fstream.tellg(); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - + bool is_file_exist(const std::string& path); + bool save_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_time(const std::string& path_to_file, time_t& ft); + bool set_file_time(const std::string& path_to_file, const time_t& ft); + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000); + bool append_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_size(const std::string& path_to_file, uint64_t &size); } } diff --git a/contrib/epee/include/memwipe.h b/contrib/epee/include/memwipe.h index c6e8f072c..c6f247913 100644 --- a/contrib/epee/include/memwipe.h +++ b/contrib/epee/include/memwipe.h @@ -32,6 +32,7 @@ #ifdef __cplusplus #include <array> +#include <cstddef> extern "C" { #endif diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 5fffde8d5..522cdf263 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -47,7 +47,7 @@ #endif #include <iostream> -#include <boost/lexical_cast.hpp> +#include <ctime> #pragma once namespace epee @@ -115,15 +115,7 @@ namespace misc_utils } - - inline std::string get_thread_string_id() - { -#if defined(_WIN32) - return boost::lexical_cast<std::string>(GetCurrentThreadId()); -#elif defined(__GNUC__) - return boost::lexical_cast<std::string>(pthread_self()); -#endif - } + std::string get_thread_string_id(); inline bool get_gmt_time(time_t t, struct tm &tm) { diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 61e2b30fe..89971bea2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -42,7 +42,7 @@ #include <boost/make_shared.hpp> #include <boost/thread.hpp> #include "warnings.h" -#include "string_tools.h" +#include "string_tools_lexical.h" #include "misc_language.h" #include "net/local_ip.h" #include "pragma_comp_defs.h" diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index bf6589c92..4af4da790 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -27,14 +27,13 @@ #pragma once -#include <boost/lexical_cast.hpp> -#include <boost/regex.hpp> +#include "memwipe.h" + #include <boost/utility/string_ref.hpp> + #include <string> #include <utility> - -#include "memwipe.h" -#include "string_tools.h" +#include <list> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" @@ -66,34 +65,9 @@ namespace net_utils typedef std::list<std::pair<std::string, std::string> > fields_list; - inline - std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) - { - fields_list::const_iterator it = fields.begin(); - for(; it != fields.end(); it++) - if(!string_tools::compare_no_case(param_name, it->first)) - break; - - if(it==fields.end()) - return std::string(); + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields); - return it->second; - } - - - inline - std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) - { - std::string buff = "([\\?|&])"; - buff += param_name + "=([^&]*)"; - boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) - { - return result[2]; - } - return std::string(); - } + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri); static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 3725ef079..29ef82fb1 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -30,7 +30,6 @@ #include <ctype.h> #include <boost/shared_ptr.hpp> #include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include <boost/optional/optional.hpp> #include <boost/utility/string_ref.hpp> //#include <mbstring.h> @@ -46,6 +45,7 @@ #endif #include "string_tools.h" +#include "string_tools_lexical.h" #include "reg_exp_definer.h" #include "abstract_http_client.h" #include "http_base.h" diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index fce6d4b7e..df59a6c44 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,6 +31,7 @@ #include <cstdint> +#include "byte_stream.h" #include "net_utils_base.h" #include "span.h" @@ -83,11 +84,12 @@ namespace levin #define LEVIN_PROTOCOL_VER_0 0 #define LEVIN_PROTOCOL_VER_1 1 + template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; @@ -125,12 +127,41 @@ namespace levin } } + //! Provides space for levin (p2p) header, so that payload can be sent without copy + class message_writer + { + byte_slice finalize(uint32_t command, uint32_t flags, uint32_t return_code, bool expect_response); + public: + using header = bucket_head2; + + explicit message_writer(std::size_t reserve = 8192); + + message_writer(const message_writer&) = delete; + message_writer(message_writer&&) = default; + ~message_writer() = default; + message_writer& operator=(const message_writer&) = delete; + message_writer& operator=(message_writer&&) = default; + + //! \return Size of payload (excludes header size). + std::size_t payload_size() const noexcept + { + return buffer.size() < sizeof(header) ? 0 : buffer.size() - sizeof(header); + } + + byte_slice finalize_invoke(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, true); } + byte_slice finalize_notify(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, false); } + byte_slice finalize_response(uint32_t command, uint32_t return_code) + { + return finalize(command, LEVIN_PACKET_RESPONSE, return_code, false); + } + + //! Has space for levin header until a finalize method is used + byte_stream buffer; + }; + //! \return Intialized levin header. bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; - //! \return A levin notification message. - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); - /*! Generate a dummy levin message. \param noise_bytes Total size of the returned `byte_slice`. @@ -140,12 +171,11 @@ namespace levin /*! Generate 1+ levin messages that are identical to the noise message size. - \param noise Each levin message will be identical to the size of this - message. The bytes from this message will be used for padding. + \param noise_size Each levin message will be identical to this value. \return `nullptr` if `noise.size()` is less than the levin header size. Otherwise, a levin notification message OR 2+ levin fragment messages. Each message is `noise.size()` in length. */ - byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); + byte_slice make_fragmented_notify(const std::size_t noise_size, int command, message_writer message); } } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index d062fa877..a6816cafc 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -51,6 +51,21 @@ #define MIN_BYTES_WANTED 512 #endif +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char* category) +{ + MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") + << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); +} + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "command-%u", command); + on_levin_traffic(context, initiator, sent, error, bytes, buf); +} + namespace epee { namespace levin @@ -88,11 +103,10 @@ public: uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id); template<class callback_t> - int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); @@ -122,12 +136,17 @@ class async_protocol_handler { std::string m_fragment_buffer; - bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + bool send_message(byte_slice message) { - const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); - if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + if (message.size() < sizeof(message_writer::header)) + return false; + + message_writer::header head; + std::memcpy(std::addressof(head), message.data(), sizeof(head)); + if(!m_pservice_endpoint->do_send(std::move(message))) return false; + on_levin_traffic(m_connection_context, true, true, false, head.m_cb, head.m_command); MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb << ", flags" << head.m_flags << ", r?=" << head.m_have_to_return_data @@ -523,26 +542,17 @@ public: { if(m_current_head.m_have_to_return_data) { - byte_slice return_buff; + levin::message_writer return_message{32 * 1024}; const uint32_t return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + m_current_head.m_command, buff_to_invoke, return_message.buffer, m_connection_context ); // peer_id remains unset if dropped if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) m_max_packet_size = m_config.m_max_packet_size; - bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); - head.m_return_code = SWAP32LE(return_code); - - if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) + if(!send_message(return_message.finalize_response(m_current_head.m_command, return_code))) return false; - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", flags" << head.m_flags - << ", r?=" << head.m_have_to_return_data - <<", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); @@ -619,7 +629,7 @@ public: } template<class callback_t> - bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, message_writer in_msg, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -638,7 +648,7 @@ public: if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if(!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -664,7 +674,7 @@ public: return true; } - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out) + int invoke(int command, message_writer in_msg, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -676,7 +686,7 @@ public: if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if (!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; @@ -713,25 +723,9 @@ public: return m_invoke_result_code; } - int notify(int command, const epee::span<const uint8_t> in_buff) - { - misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( - boost::bind(&async_protocol_handler::finish_outer_call, this)); - - CRITICAL_REGION_LOCAL(m_call_lock); - - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) - { - LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); - return -1; - } - - return 1; - } - - /*! Sends `message` without adding a levin header. The message must have - been created with `make_notify`, `make_noise_notify` or - `make_fragmented_notify`. See additional instructions for + /*! Sends `message` without adding a levin header. The message must have been + created with `make_noise_notify`, `make_fragmented_notify`, or + `message_writer::finalize_notify`. See additional instructions for `make_fragmented_notify`. \return 1 on success */ @@ -741,14 +735,11 @@ public: boost::bind(&async_protocol_handler::finish_outer_call, this) ); - const std::size_t length = message.size(); - if (!m_pservice_endpoint->do_send(std::move(message))) + if (!send_message(std::move(message))) { LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -838,19 +829,19 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config<t_connection_context>::invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->invoke(command, in_buff, buff_out) : r; + return LEVIN_OK == r ? aph->invoke(command, std::move(in_msg), buff_out) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> -int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config<t_connection_context>::invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->async_invoke(command, in_buff, cb, timeout) : r; + return LEVIN_OK == r ? aph->async_invoke(command, std::move(in_msg), cb, timeout) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> @@ -929,14 +920,6 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id) -{ - async_protocol_handler<t_connection_context>* aph; - int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->notify(command, in_buff) : r; -} -//------------------------------------------------------------------------------------------ -template<class t_connection_context> int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) { async_protocol_handler<t_connection_context>* aph; diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index cf637ba1d..d794dd51c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -24,12 +24,9 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - - #pragma once + #include "http_base.h" -#include "reg_exp_definer.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -38,173 +35,8 @@ namespace epee { namespace net_utils { - - inline bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) - { - enum state - { - st_param_name, - st_param_val - }; - state st = st_param_name; - std::string::const_iterator start_it = query.begin(); - std::pair<std::string, std::string> e; - for(std::string::const_iterator it = query.begin(); it != query.end(); it++) - { - switch(st) - { - case st_param_name: - if(*it == '=') - { - e.first.assign(start_it, it); - start_it = it;++start_it; - st = st_param_val; - } - break; - case st_param_val: - if(*it == '&') - { - e.second.assign(start_it, it); - start_it = it;++start_it; - params.push_back(e); - e.first.clear();e.second.clear(); - st = st_param_name; - } - break; - default: - LOG_ERROR("Unknown state " << (int)st); - return false; - } - } - if(st == st_param_name) - { - if(start_it != query.end()) - { - e.first.assign(start_it, query.end()); - params.push_back(e); - } - }else - { - if(start_it != query.end()) - e.second.assign(start_it, query.end()); - - if(e.first.size()) - params.push_back(e); - } - return true; - } - - inline - bool parse_uri(const std::string uri, http::uri_content& content) - { - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - content.m_query_params.clear(); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - - boost::smatch result; - if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); - content.m_path = uri; - return true; - } - if(result[1].matched) - { - content.m_path = result[1]; - } - if(result[3].matched) - { - content.m_query = result[3]; - } - if(result[5].matched) - { - content.m_fragment = result[5]; - } - if(content.m_query.size()) - { - parse_uri_query(content.m_query, content.m_query_params); - } - return true; - } - - inline - bool parse_url_ipv6(const std::string url_str, http::url_content& content) - { - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 3 4 5 6 7 - - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return false; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - else // if host not matched, matching should be considered failed - { - return false; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - - inline - bool parse_url(const std::string url_str, http::url_content& content) - { - - if (parse_url_ipv6(url_str, content)) return true; - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 34 5 6 7 - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return true; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - + bool parse_uri(const std::string uri, http::uri_content& content); + bool parse_url_ipv6(const std::string url_str, http::url_content& content); + bool parse_url(const std::string url_str, http::url_content& content); } } diff --git a/contrib/epee/include/reg_exp_definer.h b/contrib/epee/include/reg_exp_definer.h index eb11c9e10..386a45f9a 100644 --- a/contrib/epee/include/reg_exp_definer.h +++ b/contrib/epee/include/reg_exp_definer.h @@ -29,6 +29,7 @@ #define _REG_EXP_DEFINER_H_ #include <boost/interprocess/detail/atomic.hpp> +#include <boost/regex.hpp> #include "syncobj.h" namespace epee diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 802e16c1b..383d67cc2 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -37,21 +37,14 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category); + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command); + namespace { - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category) - { - MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") - << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); - } - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) - { - char buf[32]; - snprintf(buf, sizeof(buf), "command-%u", command); - return on_levin_traffic(context, initiator, sent, error, bytes, buf); - } static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { 8192, // objects 16384, // fields @@ -117,12 +110,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; + levin::message_writer to_send{16 * 1024}; std::string buff_to_recv; - stg.store_to_binary(buff_to_send, 16 * 1024); + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); + int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); @@ -145,10 +137,9 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send, 16 * 1024); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + levin::message_writer to_send{16 * 1024}; + stg.store_to_binary(to_send.buffer); + int res = transport.invoke_async(command, std::move(to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -192,11 +183,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send); + levin::message_writer to_send; + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); + int res = transport.send(to_send.finalize_notify(command), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -207,7 +197,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff, &default_levin_limits)) @@ -230,12 +220,11 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out, 32 * 1024)) + if(!strg_out.store_to_binary(buff_out)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; } - on_levin_traffic(context, false, true, false, buff_out.size(), command); return res; } @@ -262,7 +251,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -271,13 +260,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; epee::byte_slice fake_str; \ - return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ + bool handled = false; epee::byte_stream fake_str; \ + return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -297,7 +286,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index c5d0c48ee..655a2eb12 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -34,6 +34,7 @@ namespace epee { class byte_slice; + class byte_stream; namespace serialization { /************************************************************************/ @@ -83,8 +84,13 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); - bool load_from_binary(const std::string& target, const limits_t *limits = NULL); + bool store_to_binary(byte_stream& ss); + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = nullptr); + bool load_from_binary(const std::string& target, const limits_t *limits = nullptr) + { + return load_from_binary(epee::strspan<uint8_t>(target), limits); + } + template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index e7250e895..7f6596f36 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -29,13 +29,15 @@ #include <string> #include "byte_slice.h" -#include "parserse_base_utils.h" +#include "parserse_base_utils.h" /// TODO: (mj-xmr) This will be reduced in an another PR #include "portable_storage.h" #include "file_io_utils.h" #include "span.h" namespace epee { + class byte_stream; + namespace serialization { //----------------------------------------------------------------------------------------------------------- @@ -127,5 +129,14 @@ namespace epee store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } + //----------------------------------------------------------------------------------------------------------- + template<class t_struct> + bool store_t_to_binary(t_struct& str_in, byte_stream& binary_buff) + { + portable_storage ps; + str_in.store(ps); + return ps.store_to_binary(binary_buff); + } + } } diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 2d9317d60..dbbe1906e 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -24,33 +24,16 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #ifndef _STRING_TOOLS_H_ #define _STRING_TOOLS_H_ -// Previously pulled in by ASIO, further cleanup still required ... -#ifdef _WIN32 -# include <winsock2.h> -# include <windows.h> -#endif - -#include <string.h> -#include <locale> -#include <cstdlib> -#include <string> -#include <type_traits> -#include <boost/lexical_cast.hpp> -#include <boost/algorithm/string/predicate.hpp> -#include <boost/utility/string_ref.hpp> -#include "misc_log_ex.h" -#include "storages/parserse_base_utils.h" #include "hex.h" -#include "memwipe.h" #include "mlocker.h" -#include "span.h" -#include "warnings.h" +#include <boost/utility/string_ref.hpp> +#include <sstream> +#include <string> +#include <cstdint> #ifndef OUT #define OUT @@ -74,202 +57,38 @@ namespace string_tools { return from_hex::to_string(res, s); } - //---------------------------------------------------------------------------- -PUSH_WARNINGS -DISABLE_GCC_WARNING(maybe-uninitialized) - template<class XType> - inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) - { - if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) - { - for (char c : str_id) - { - if (!epee::misc_utils::parse::isdigit(c)) - return false; - } - } - - try - { - val = boost::lexical_cast<XType>(str_id); - return true; - } - catch(const std::exception& /*e*/) - { - //const char* pmsg = e.what(); - return false; - } - catch(...) - { - return false; - } - - return true; - } -POP_WARNINGS - //---------------------------------------------------------------------------- - template<class XType> - inline bool xtype_to_string(const XType& val, std::string& str) - { - try - { - str = boost::lexical_cast<std::string>(val); - } - catch(...) - { - return false; - } - - return true; - } - //---------------------------------------------------------------------------- - std::string get_ip_string_from_int32(uint32_t ip); - //---------------------------------------------------------------------------- - bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); - //---------------------------------------------------------------------------- - inline bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) - { - //parse ip and address - std::string::size_type p = addres.find(':'); - std::string ip_str, port_str; - if(p == std::string::npos) - { - port = 0; - ip_str = addres; - } - else - { - ip_str = addres.substr(0, p); - port_str = addres.substr(p+1, addres.size()); - } - - if(!get_ip_int32_from_string(ip, ip_str)) - { - return false; - } - - if(p != std::string::npos && !get_xtype_from_string(port, port_str)) - { - return false; - } - return true; - } - - inline std::string num_to_string_fast(int64_t val) - { - /* - char buff[30] = {0}; - i64toa_s(val, buff, sizeof(buff)-1, 10); - return buff;*/ - return boost::lexical_cast<std::string>(val); - } - //---------------------------------------------------------------------------- - template<typename T> - inline std::string to_string_hex(const T &val) - { - static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); - std::stringstream ss; - ss << std::hex << val; - std::string s; - ss >> s; - return s; - } - //---------------------------------------------------------------------------- + + std::string get_ip_string_from_int32(uint32_t ip); + bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres); + std::string num_to_string_fast(int64_t val); - inline bool compare_no_case(const std::string& str1, const std::string& str2) - { - - return !boost::iequals(str1, str2); - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_name() - { - static std::string module_name; - return module_name; - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_folder() - { - static std::string module_folder; - return module_folder; - } - //---------------------------------------------------------------------------- + bool compare_no_case(const std::string& str1, const std::string& str2); + std::string& get_current_module_name(); + std::string& get_current_module_folder(); #ifdef _WIN32 - inline std::string get_current_module_path() - { - char pname [5000] = {0}; - GetModuleFileNameA( NULL, pname, sizeof(pname)); - pname[sizeof(pname)-1] = 0; //be happy ;) - return pname; - } + std::string get_current_module_path(); #endif - //---------------------------------------------------------------------------- - inline bool set_module_name_and_folder(const std::string& path_to_process_) - { - std::string path_to_process = path_to_process_; -#ifdef _WIN32 - path_to_process = get_current_module_path(); -#endif - std::string::size_type a = path_to_process.rfind( '\\' ); - if(a == std::string::npos ) - { - a = path_to_process.rfind( '/' ); - } - if ( a != std::string::npos ) - { - get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); - get_current_module_folder() = path_to_process.substr(0, a); - return true; - }else - return false; - - } - - //---------------------------------------------------------------------------- - inline bool trim_left(std::string& str) - { - for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) - str.erase(str.begin()); - - return true; - } - //---------------------------------------------------------------------------- - inline bool trim_right(std::string& str) - { - - for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) - str.erase( --((it++).base())); - - return true; - } - //---------------------------------------------------------------------------- - inline std::string& trim(std::string& str) - { - - trim_left(str); - trim_right(str); - return str; - } + bool set_module_name_and_folder(const std::string& path_to_process_); + bool trim_left(std::string& str); + bool trim_right(std::string& str); //---------------------------------------------------------------------------- - inline std::string trim(const std::string& str_) + inline std::string& trim(std::string& str) { - std::string str = str_; trim_left(str); trim_right(str); return str; } //---------------------------------------------------------------------------- - inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false) + inline std::string trim(const std::string& str_) { - if (s.size() < n) - { - if (prepend) - s = std::string(n - s.size(), c) + s; - else - s.append(n - s.size(), c); - } - return s; + std::string str = str_; + trim_left(str); + trim_right(str); + return str; } + std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false); + //---------------------------------------------------------------------------- template<class t_pod_type> std::string pod_to_hex(const t_pod_type& s) @@ -296,64 +115,25 @@ POP_WARNINGS { return hex_to_pod(hex_str, unwrap(s)); } - //---------------------------------------------------------------------------- - bool validate_hex(uint64_t length, const std::string& str); - //---------------------------------------------------------------------------- - inline std::string get_extension(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return res; - - res = str.substr(pos+1, str.size()-pos); - return res; - } - //---------------------------------------------------------------------------- - inline std::string cut_off_extension(const std::string& str) + //---------------------------------------------------------------------------- + template<typename T> + inline std::string to_string_hex(const T &val) { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return str; - - res = str.substr(0, pos); - return res; + static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); + std::stringstream ss; + ss << std::hex << val; + std::string s; + ss >> s; + return s; } - //---------------------------------------------------------------------------- + + bool validate_hex(uint64_t length, const std::string& str); + std::string get_extension(const std::string& str); + std::string cut_off_extension(const std::string& str); + #ifdef _WIN32 - inline std::wstring utf8_to_utf16(const std::string& str) - { - if (str.empty()) - return {}; - int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); - if (wstr_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::wstring wstr(wstr_size, wchar_t{}); - if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return wstr; - } - inline std::string utf16_to_utf8(const std::wstring& wstr) - { - if (wstr.empty()) - return {}; - int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); - if (str_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::string str(str_size, char{}); - if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return str; - } + std::wstring utf8_to_utf16(const std::string& str); + std::string utf16_to_utf8(const std::wstring& wstr); #endif } } diff --git a/contrib/epee/include/string_tools_lexical.h b/contrib/epee/include/string_tools_lexical.h new file mode 100644 index 000000000..01be562bc --- /dev/null +++ b/contrib/epee/include/string_tools_lexical.h @@ -0,0 +1,91 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef _STRING_TOOLS_LEXICAL_H_ +#define _STRING_TOOLS_LEXICAL_H_ + +#include "warnings.h" +#include "storages/parserse_base_utils.h" +#include <boost/lexical_cast.hpp> // A heavy header, that was extracted from the rest + +#ifndef OUT + #define OUT +#endif + +namespace epee +{ +namespace string_tools +{ +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) + template<class XType> + inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) + { + if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) + { + for (char c : str_id) + { + if (!epee::misc_utils::parse::isdigit(c)) + return false; + } + } + + try + { + val = boost::lexical_cast<XType>(str_id); + return true; + } + catch(const std::exception& /*e*/) + { + //const char* pmsg = e.what(); + return false; + } + catch(...) + { + return false; + } + + return true; + } +POP_WARNINGS + + template<class XType> + inline bool xtype_to_string(const XType& val, std::string& str) + { + try + { + str = boost::lexical_cast<std::string>(val); + } + catch(...) + { + return false; + } + + return true; + } +} +} +#endif //_STRING_TOOLS_LEXICAL_H_ diff --git a/contrib/epee/include/tiny_ini.h b/contrib/epee/include/tiny_ini.h index 2bc71fc1a..6ced548eb 100644 --- a/contrib/epee/include/tiny_ini.h +++ b/contrib/epee/include/tiny_ini.h @@ -28,8 +28,6 @@ #ifndef _TINY_INI_H_ #define _TINY_INI_H_ -#include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include "string_tools.h" namespace epee @@ -37,20 +35,8 @@ namespace epee namespace tiny_ini { - inline - bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) - { - std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; - const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) - return false; - res = result[2]; - string_tools::trim(res); - return true; - } - inline - std::string get_param_value(const std::string& param_name, const std::string& ini_entry) + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res); + inline std::string get_param_value(const std::string& param_name, const std::string& ini_entry) { std::string buff; get_param_value(param_name, ini_entry, buff); diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index f89d6f8cc..0f0a6ecad 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -35,7 +35,11 @@ add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_cli wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp int-util.cpp portable_storage.cpp misc_language.cpp - + misc_os_dependent.cpp + file_io_utils.cpp + net_parse_helpers.cpp + http_base.cpp + tiny_ini.cpp ${EPEE_HEADERS_PUBLIC} ) diff --git a/contrib/epee/src/abstract_http_client.cpp b/contrib/epee/src/abstract_http_client.cpp index 540917873..3ae09c90e 100644 --- a/contrib/epee/src/abstract_http_client.cpp +++ b/contrib/epee/src/abstract_http_client.cpp @@ -1,6 +1,7 @@ #include "net/abstract_http_client.h" #include "net/http_base.h" #include "net/net_parse_helpers.h" +#include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" diff --git a/contrib/epee/src/file_io_utils.cpp b/contrib/epee/src/file_io_utils.cpp new file mode 100644 index 000000000..5072adcbc --- /dev/null +++ b/contrib/epee/src/file_io_utils.cpp @@ -0,0 +1,231 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "file_io_utils.h" + +#include <fstream> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> +#ifdef WIN32 +#include <windows.h> +#include "string_tools.h" +#endif + +// On Windows there is a problem with non-ASCII characters in path and file names +// as far as support by the standard components used is concerned: + +// The various file stream classes, e.g. std::ifstream and std::ofstream, are +// part of the GNU C++ Library / libstdc++. On the most basic level they use the +// fopen() call as defined / made accessible to programs compiled within MSYS2 +// by the stdio.h header file maintained by the MinGW project. + +// The critical point: The implementation of fopen() is part of MSVCRT, the +// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any +// Unicode support. + +// Monero code that would want to continue to use the normal file stream classes +// but WITH Unicode support could therefore not solve this problem on its own, +// but 2 different projects from 2 different maintaining groups would need changes +// in this particular direction - something probably difficult to achieve and +// with a long time to wait until all new versions / releases arrive. + +// Implemented solution approach: Circumvent the problem by stopping to use std +// file stream classes on Windows and directly use Unicode-capable WIN32 API +// calls. Most of the code doing so is concentrated in this header file here. + +namespace epee +{ +namespace file_io_utils +{ + + bool is_file_exist(const std::string& path) + { + boost::filesystem::path p(path); + return boost::filesystem::exists(p); + } + + + bool save_string_to_file(const std::string& path_to_file, const std::string& str) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD bytes_written; + DWORD bytes_to_write = (DWORD)str.size(); + BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); + CloseHandle(file_handle); + if (bytes_written != bytes_to_write) + result = FALSE; + return result; +#else + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool get_file_time(const std::string& path_to_file, time_t& ft) + { + boost::system::error_code ec; + ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); + if(!ec) + return true; + else + return false; + } + + + bool set_file_time(const std::string& path_to_file, const time_t& ft) + { + boost::system::error_code ec; + boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); + if(!ec) + return true; + else + return false; + } + + + + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD file_size = GetFileSize(file_handle, NULL); + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { + CloseHandle(file_handle); + return false; + } + target_str.resize(file_size); + DWORD bytes_read; + BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); + CloseHandle(file_handle); + if (bytes_read != file_size) + result = FALSE; + return result; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + + std::ifstream::pos_type file_size = fstream.tellg(); + + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large + return false;//don't go crazy + size_t file_size_t = static_cast<size_t>(file_size); + + target_str.resize(file_size_t); + + fstream.seekg (0, std::ios::beg); + fstream.read((char*)target_str.data(), target_str.size()); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool append_string_to_file(const std::string& path_to_file, const std::string& str) + { + // No special Windows implementation because so far not used in Monero code + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } + } + + + bool get_file_size(const std::string& path_to_file, uint64_t &size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER file_size; + BOOL result = GetFileSizeEx(file_handle, &file_size); + CloseHandle(file_handle); + if (result) { + size = file_size.QuadPart; + } + return size; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + size = fstream.tellg(); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + +} +} diff --git a/contrib/epee/src/http_base.cpp b/contrib/epee/src/http_base.cpp new file mode 100644 index 000000000..647dfb899 --- /dev/null +++ b/contrib/epee/src/http_base.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/http_base.h" +#include "memwipe.h" +#include "string_tools.h" + +#include <boost/regex.hpp> +#include <string> +#include <utility> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.http" + +namespace epee +{ +namespace net_utils +{ +namespace http +{ + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) + { + fields_list::const_iterator it = fields.begin(); + for(; it != fields.end(); it++) + if(!string_tools::compare_no_case(param_name, it->first)) + break; + + if(it==fields.end()) + return std::string(); + + return it->second; + } + + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) + { + std::string buff = "([\\?|&])"; + buff += param_name + "=([^&]*)"; + boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) + { + return result[2]; + } + return std::string(); + } +} +} +} diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp index 5ec86b3d6..7c5cd5a78 100644 --- a/contrib/epee/src/levin_base.cpp +++ b/contrib/epee/src/levin_base.cpp @@ -34,6 +34,25 @@ namespace epee { namespace levin { + message_writer::message_writer(const std::size_t reserve) + : buffer() + { + buffer.reserve(reserve); + buffer.put_n(0, sizeof(header)); + } + + byte_slice message_writer::finalize(const uint32_t command, const uint32_t flags, const uint32_t return_code, const bool expect_response) + { + if (buffer.size() < sizeof(header)) + throw std::runtime_error{"levin_writer::finalize already called"}; + + header head = make_header(command, payload_size(), flags, expect_response); + head.m_return_code = SWAP32LE(return_code); + + std::memcpy(buffer.tellp() - buffer.size(), std::addressof(head), sizeof(head)); + return byte_slice{std::move(buffer)}; + } + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept { bucket_head2 head = {0}; @@ -47,12 +66,6 @@ namespace levin return head; } - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) - { - const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); - return byte_slice{epee::as_byte_span(head), payload}; - } - byte_slice make_noise_notify(const std::size_t noise_bytes) { static constexpr const std::uint32_t flags = @@ -68,46 +81,40 @@ namespace levin return byte_slice{std::move(buffer)}; } - byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + byte_slice make_fragmented_notify(const std::size_t noise_size, const int command, message_writer message) { - const size_t noise_size = noise_message.size(); if (noise_size < sizeof(bucket_head2) * 2) return nullptr; - if (payload.size() <= noise_size - sizeof(bucket_head2)) + if (message.buffer.size() <= noise_size) { /* The entire message can be sent at once, and the levin binary parser will ignore extra bytes. So just pad with zeroes and otherwise send a "normal", not fragmented message. */ - const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); - const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; - const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); - return byte_slice{as_byte_span(head), payload, padding_bytes}; + message.buffer.put_n(0, noise_size - message.buffer.size()); + return message.finalize_notify(command); } // fragment message + const byte_slice payload_bytes = message.finalize_notify(command); + span<const std::uint8_t> payload = to_span(payload_bytes); + const size_t payload_space = noise_size - sizeof(bucket_head2); const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; - std::string buffer{}; - buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + byte_stream buffer{}; + buffer.reserve(expected_fragments * noise_size); - bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + bucket_head2 head = make_header(0, payload_space, LEVIN_PACKET_BEGIN, false); + buffer.write(as_byte_span(head)); - head.m_command = command; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_cb = payload.size(); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + // internal levin header is in payload already - size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + size_t copy_size = payload.remove_prefix(payload_space); + buffer.write(payload.data() - copy_size, copy_size); - head.m_command = 0; head.m_flags = 0; - head.m_cb = noise_size - sizeof(bucket_head2); - while (!payload.empty()) { copy_size = payload.remove_prefix(payload_space); @@ -115,12 +122,12 @@ namespace levin if (payload.empty()) head.m_flags = LEVIN_PACKET_END; - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + buffer.write(as_byte_span(head)); + buffer.write(payload.data() - copy_size, copy_size); } const size_t padding = noise_size - copy_size - sizeof(bucket_head2); - buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + buffer.put_n(0, padding); return byte_slice{std::move(buffer)}; } diff --git a/contrib/epee/src/misc_os_dependent.cpp b/contrib/epee/src/misc_os_dependent.cpp new file mode 100644 index 000000000..cd4967131 --- /dev/null +++ b/contrib/epee/src/misc_os_dependent.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_os_dependent.h" +#include <boost/lexical_cast.hpp> + +namespace epee +{ +namespace misc_utils +{ + // TODO: (vtnerd) This function is weird since boost::this_thread::get_id() exists but returns a different value. + std::string get_thread_string_id() + { +#if defined(_WIN32) + return boost::lexical_cast<std::string>(GetCurrentThreadId()); +#elif defined(__GNUC__) + return boost::lexical_cast<std::string>(pthread_self()); +#endif + } +} +} diff --git a/contrib/epee/src/net_parse_helpers.cpp b/contrib/epee/src/net_parse_helpers.cpp new file mode 100644 index 000000000..de7843b67 --- /dev/null +++ b/contrib/epee/src/net_parse_helpers.cpp @@ -0,0 +1,206 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/net_parse_helpers.h" +#include "net/http_base.h" +#include "misc_log_ex.h" +#include "reg_exp_definer.h" +#include <boost/lexical_cast.hpp> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net" + +namespace epee +{ +namespace net_utils +{ + + static bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) + { + enum state + { + st_param_name, + st_param_val + }; + state st = st_param_name; + std::string::const_iterator start_it = query.begin(); + std::pair<std::string, std::string> e; + for(std::string::const_iterator it = query.begin(); it != query.end(); it++) + { + switch(st) + { + case st_param_name: + if(*it == '=') + { + e.first.assign(start_it, it); + start_it = it;++start_it; + st = st_param_val; + } + break; + case st_param_val: + if(*it == '&') + { + e.second.assign(start_it, it); + start_it = it;++start_it; + params.push_back(e); + e.first.clear();e.second.clear(); + st = st_param_name; + } + break; + default: + LOG_ERROR("Unknown state " << (int)st); + return false; + } + } + if(st == st_param_name) + { + if(start_it != query.end()) + { + e.first.assign(start_it, query.end()); + params.push_back(e); + } + }else + { + if(start_it != query.end()) + e.second.assign(start_it, query.end()); + + if(e.first.size()) + params.push_back(e); + } + return true; + } + + bool parse_uri(const std::string uri, http::uri_content& content) + { + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + content.m_query_params.clear(); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + + boost::smatch result; + if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); + content.m_path = uri; + return true; + } + if(result[1].matched) + { + content.m_path = result[1]; + } + if(result[3].matched) + { + content.m_query = result[3]; + } + if(result[5].matched) + { + content.m_fragment = result[5]; + } + if(content.m_query.size()) + { + parse_uri_query(content.m_query, content.m_query_params); + } + return true; + } + + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 + + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + bool parse_url(const std::string url_str, http::url_content& content) + { + + if (parse_url_ipv6(url_str, content)) return true; + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 34 5 6 7 + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return true; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + +} +} diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index c3c9ccc02..b922cc9e3 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -48,15 +48,23 @@ namespace serialization TRY_ENTRY(); byte_stream ss; ss.reserve(initial_buffer_size); + store_to_binary(ss); + target = epee::byte_slice{std::move(ss)}; + return true; + CATCH_ENTRY("portable_storage::store_to_binary", false); + } + + bool portable_storage::store_to_binary(byte_stream& ss) + { + TRY_ENTRY(); storage_block_header sbh{}; sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; ss.write(epee::as_byte_span(sbh)); pack_entry_to_buff(ss, m_root); - target = epee::byte_slice{std::move(ss)}; return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) + CATCH_ENTRY("portable_storage::store_to_binary", false); } bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) @@ -76,11 +84,6 @@ namespace serialization CATCH_ENTRY("portable_storage::load_from_json", false) } - bool portable_storage::load_from_binary(const std::string& target, const limits_t *limits) - { - return load_from_binary(epee::strspan<uint8_t>(target), limits); - } - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) { m_root.m_entries.clear(); diff --git a/contrib/epee/src/string_tools.cpp b/contrib/epee/src/string_tools.cpp index fd0254016..984a151b5 100644 --- a/contrib/epee/src/string_tools.cpp +++ b/contrib/epee/src/string_tools.cpp @@ -25,6 +25,29 @@ // #include "string_tools.h" +#include "string_tools_lexical.h" + + +// Previously pulled in by ASIO, further cleanup still required ... +#ifdef _WIN32 +# include <winsock2.h> +# include <windows.h> +#endif + +#include <locale> +#include <cstdlib> +#include <string> +#include <type_traits> +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/utility/string_ref.hpp> +#include "misc_log_ex.h" +#include "storages/parserse_base_utils.h" +#include "hex.h" +#include "memwipe.h" +#include "mlocker.h" +#include "span.h" +#include "warnings.h" #include <ctype.h> @@ -68,6 +91,180 @@ namespace string_tools return false; return true; } + //---------------------------------------------------------------------------- + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) + { + //parse ip and address + std::string::size_type p = addres.find(':'); + std::string ip_str, port_str; + if(p == std::string::npos) + { + port = 0; + ip_str = addres; + } + else + { + ip_str = addres.substr(0, p); + port_str = addres.substr(p+1, addres.size()); + } + + if(!get_ip_int32_from_string(ip, ip_str)) + { + return false; + } + + if(p != std::string::npos && !get_xtype_from_string(port, port_str)) + { + return false; + } + return true; + } + + std::string num_to_string_fast(int64_t val) + { + /* + char buff[30] = {0}; + i64toa_s(val, buff, sizeof(buff)-1, 10); + return buff;*/ + return boost::lexical_cast<std::string>(val); + } + + + bool compare_no_case(const std::string& str1, const std::string& str2) + { + + return !boost::iequals(str1, str2); + } + //---------------------------------------------------------------------------- + std::string& get_current_module_name() + { + static std::string module_name; + return module_name; + } + //---------------------------------------------------------------------------- + std::string& get_current_module_folder() + { + static std::string module_folder; + return module_folder; + } + +#ifdef _WIN32 + std::string get_current_module_path() + { + char pname [5000] = {0}; + GetModuleFileNameA( NULL, pname, sizeof(pname)); + pname[sizeof(pname)-1] = 0; //be happy ;) + return pname; + } +#endif + + bool set_module_name_and_folder(const std::string& path_to_process_) + { + std::string path_to_process = path_to_process_; +#ifdef _WIN32 + path_to_process = get_current_module_path(); +#endif + std::string::size_type a = path_to_process.rfind( '\\' ); + if(a == std::string::npos ) + { + a = path_to_process.rfind( '/' ); + } + if ( a != std::string::npos ) + { + get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); + get_current_module_folder() = path_to_process.substr(0, a); + return true; + }else + return false; + + } + + //---------------------------------------------------------------------------- + bool trim_left(std::string& str) + { + for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) + str.erase(str.begin()); + + return true; + } + //---------------------------------------------------------------------------- + bool trim_right(std::string& str) + { + + for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) + str.erase( --((it++).base())); + + return true; + } + //---------------------------------------------------------------------------- + std::string pad_string(std::string s, size_t n, char c, bool prepend) + { + if (s.size() < n) + { + if (prepend) + s = std::string(n - s.size(), c) + s; + else + s.append(n - s.size(), c); + } + return s; + } + + std::string get_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return res; + + res = str.substr(pos+1, str.size()-pos); + return res; + } + //---------------------------------------------------------------------------- + std::string cut_off_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return str; + + res = str.substr(0, pos); + return res; + } + //---------------------------------------------------------------------------- +#ifdef _WIN32 + std::wstring utf8_to_utf16(const std::string& str) + { + if (str.empty()) + return {}; + int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); + if (wstr_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::wstring wstr(wstr_size, wchar_t{}); + if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return wstr; + } + std::string utf16_to_utf8(const std::wstring& wstr) + { + if (wstr.empty()) + return {}; + int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); + if (str_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::string str(str_size, char{}); + if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return str; + } +#endif } } diff --git a/contrib/epee/src/tiny_ini.cpp b/contrib/epee/src/tiny_ini.cpp new file mode 100644 index 000000000..577ebf7c6 --- /dev/null +++ b/contrib/epee/src/tiny_ini.cpp @@ -0,0 +1,46 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "string_tools.h" +#include <boost/regex.hpp> + +namespace epee +{ +namespace tiny_ini +{ + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) + { + std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; + const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) + return false; + res = result[2]; + string_tools::trim(res); + return true; + } +} +} diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index e2e1d0b94..0a40d4608 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -36,6 +36,10 @@ This guide explains how to set up the environment, and how to start the builds. You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. +```bash +sudo adduser gitianuser +sudo usermod -aG sudo gitianuser +``` LXC --- @@ -83,7 +87,7 @@ Docker Prepare for building with docker: ```bash -sudo apt-get install git make curl docker.io +sudo bash -c 'apt-get update && apt-get upgrade -y && apt-get install git curl docker.io' ``` Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): @@ -96,13 +100,12 @@ sudo usermod -aG docker gitianuser Optionally add yourself to the docker group. Note that this will give docker root access to your system. ```bash -sudo usermod -aG docker gitianuser +sudo usermod -aG docker $USER ``` Manual Building ------------------- -The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. ======= The script automatically installs some packages with apt. If you are not running it on a debian-like system, pass `--no-apt` along with the other arguments to it. It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. @@ -122,17 +125,23 @@ cp monero/contrib/gitian/gitian-build.py . ### Setup the required environment -Setup for LXC: +Common setup part: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 +su - gitianuser -./gitian-build.py --setup $GH_USER $VERSION +GH_USER=YOUR_GITHUB_USER_NAME +VERSION=v0.17.2.0 ``` Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. +Setup for LXC: + +```bash +./gitian-build.py --setup $GH_USER $VERSION +``` + Setup for docker: ```bash @@ -145,8 +154,10 @@ fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) or pass the signed assert file back to your build machine. ```bash -git clone git@github.com:monero-project/gitian.sigs.git -git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git +git clone https://github.com/monero-project/gitian.sigs/ +pushd gitian.sigs +git remote add $GH_USER https://github.com/$GH_USER/gitian.sigs +popd ``` Build the binaries @@ -154,13 +165,26 @@ Build the binaries **Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. +Currently working MacOS solution: + +```bash +curl -O https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz +mv MacOSX10.11.sdk.tar.gz builder/inputs +``` + To build the most recent tag (pass in `--docker` if using docker): ```bash ./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION ``` -To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. +To speed up the build, use `-j 5 --memory 10000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 10000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. A good rule of thumb is, that Monero currently needs about 2 GB of RAM per core. + +A full example for `docker` would look like the following: + +```bash +./gitian-build.py -j 5 --memory 10000 --docker --detach-sign --no-commit --build $GH_USER $VERSION +``` If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. @@ -171,6 +195,22 @@ Take a look in the assert files and note the SHA256 checksums listed there. You should verify that the checksum that is listed matches each of the binaries you actually built. This may be done on Linux using the `sha256sum` command or on MacOS using `shasum --algorithm 256` for example. +An example script to verify the checksums would be: + +```bash +pushd out/${VERSION} + +for ASSERT in ../../sigs/${VERSION}-*/*/*.assert; do + if ! sha256sum --ignore-missing -c "${ASSERT}" ; then + echo "FAILED for ${ASSERT} ! Please inspect manually." + fi +done + +popd +``` + +Don't ignore the incorrect formatting of the found assert files. These files you'll have to compare manually (currently OSX and FreeBSD). + You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. @@ -181,14 +221,7 @@ Signing assert files If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 - -gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert -gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert -gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert -gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert -gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert +for ASSERT in sigs/${VERSION}-*/*/*.assert; do gpg --detach-sign ${ASSERT}; done ``` This will create a `.sig` file for each `.assert` file above (2 files for each platform). @@ -201,6 +234,7 @@ Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash +cd gitian.sigs git checkout -b $VERSION # add your assert and sig files... git commit -S -a -m "Add $GH_USER $VERSION" diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index a8140a8a6..6bf936958 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -94,10 +94,6 @@ def build(): os.chdir('builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) - subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) - subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', 'inputs/monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) rebuild() diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 96c741f66..6314d5775 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -4882,9 +4882,6 @@ mdb_env_open2(MDB_env *env, int prev) #endif env->me_maxpg = env->me_mapsize / env->me_psize; - if (env->me_txns) - env->me_txns->mti_txnid = meta.mm_txnid; - #if MDB_DEBUG { MDB_meta *meta = mdb_env_pick_meta(env); @@ -4984,6 +4981,9 @@ static int ESECT mdb_env_share_locks(MDB_env *env, int *excl) { int rc = 0; + MDB_meta *meta = mdb_env_pick_meta(env); + + env->me_txns->mti_txnid = meta->mm_txnid; #ifdef _WIN32 { diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 35fb86552..fcda54547 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -31,6 +31,7 @@ cmake_minimum_required(VERSION 2.8.7) project(easylogging CXX) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +monero_enable_coverage() find_package(Threads) find_package(Backtrace) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index caaf7944c..f1722f0a1 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2490,20 +2490,20 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { template<typename Transform> -static inline std::string utf8canonical(const std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) +static inline void utf8canonical(std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) { - std::string sc = ""; size_t avail = s.size(); const char *ptr = s.data(); wint_t cp = 0; - int bytes = 1; + int rbytes = 1, bytes = -1; char wbuf[8], *wptr; + size_t w_offset = 0; while (avail--) { if ((*ptr & 0x80) == 0) { cp = *ptr++; - bytes = 1; + rbytes = 1; } else if ((*ptr & 0xe0) == 0xc0) { @@ -2512,7 +2512,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp = (*ptr++ & 0x1f) << 6; cp |= *ptr++ & 0x3f; --avail; - bytes = 2; + rbytes = 2; } else if ((*ptr & 0xf0) == 0xe0) { @@ -2522,7 +2522,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp |= (*ptr++ & 0x3f) << 6; cp |= *ptr++ & 0x3f; avail -= 2; - bytes = 3; + rbytes = 3; } else if ((*ptr & 0xf8) == 0xf0) { @@ -2533,7 +2533,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp |= (*ptr++ & 0x3f) << 6; cp |= *ptr++ & 0x3f; avail -= 3; - bytes = 4; + rbytes = 4; } else throw std::runtime_error("Invalid UTF-8"); @@ -2550,6 +2550,9 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w else throw std::runtime_error("Invalid code point UTF-8 transformation"); + if (bytes > rbytes) + throw std::runtime_error("In place sanitization requires replacements to not take more space than the original code points"); + wptr = wbuf; switch (bytes) { @@ -2560,16 +2563,17 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w default: throw std::runtime_error("Invalid UTF-8"); } *wptr = 0; - sc.append(wbuf, bytes); + memcpy(&s[w_offset], wbuf, bytes); + w_offset += bytes; cp = 0; bytes = 1; } - return sc; + s.resize(w_offset); } void sanitize(std::string &s) { - s = utf8canonical(s, [](wint_t c)->wint_t { + utf8canonical(s, [](wint_t c)->wint_t { if (c == 9 || c == 10 || c == 13) return c; if (c < 0x20) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index b1b238427..99a84606d 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> +#include <boost/filesystem/path.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index e439895bf..18a37434c 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 6199586bf..8c3c3a009 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem/path.hpp> #include <boost/algorithm/string.hpp> #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 9a9d58c46..b1c599f3a 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -29,6 +29,8 @@ #include <array> #include <lmdb.h> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/pruning.h" #include "cryptonote_core/cryptonote_core.h" diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index 1a54778a7..78a662134 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "serialization/crypto.h" #include "cryptonote_core/tx_pool.h" diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 1f728b4e5..5f5ca6abf 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_basic/cryptonote_boost_serialization.h" diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 095e6c503..8356ef420 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem/path.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_core/tx_pool.h" diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index c88a630cc..6b48d8723 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -34,6 +34,8 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <functional> #include <vector> diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index ebc367b3a..051220ee1 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -35,6 +35,10 @@ #include "common/i18n.h" #include "translation_files.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> +#include <algorithm> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 3e4532d4e..b0c4a25d8 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include "wipeable_string.h" #include "string_tools.h" +#include "string_tools_lexical.h" #include "serialization/string.h" #include "cryptonote_format_utils.h" #include "cryptonote_config.h" diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 29f6dce5a..ae514aac6 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -44,6 +44,7 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" #include "boost/logic/tribool.hpp" +#include <boost/filesystem.hpp> #ifdef __APPLE__ #include <sys/times.h> diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index cdad39a2c..a3d695b85 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -57,6 +57,7 @@ #include "common/notify.h" #include "common/varint.h" #include "common/pruning.h" +#include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 57104fd59..beae55b35 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -57,6 +57,8 @@ using namespace epee; #include "hardforks/hardforks.h" #include "version.h" +#include <boost/filesystem.hpp> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 28530f3e7..80dd2bc39 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -46,6 +46,8 @@ #include "block_queue.h" #include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" +#include "net/levin_base.h" +#include "p2p/net_node_common.h" #include <boost/circular_buffer.hpp> PUSH_WARNINGS @@ -195,10 +197,11 @@ namespace cryptonote bool post_notify(typename t_parameter::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parameter).name() << " -->"); - epee::byte_slice blob; - epee::serialization::store_t_to_binary(arg, blob, 256 * 1024); // optimize for block responses + + epee::levin::message_writer out{256 * 1024}; // optimize for block responses + epee::serialization::store_t_to_binary(arg, out.buffer); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::to_span(blob), context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, std::move(out), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 69ef0970b..afc81f552 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -2713,15 +2713,15 @@ skip: // send fluffy ones first, we want to encourage people to run that if (!fluffyConnections.empty()) { - epee::byte_slice fluffyBlob; - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob, 32 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::to_span(fluffyBlob), std::move(fluffyConnections)); + epee::levin::message_writer fluffyBlob{32 * 1024}; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, std::move(fluffyBlob), std::move(fluffyConnections)); } if (!fullConnections.empty()) { - epee::byte_slice fullBlob; - epee::serialization::store_t_to_binary(arg, fullBlob, 128 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::to_span(fullBlob), std::move(fullConnections)); + epee::levin::message_writer fullBlob{128 * 1024}; + epee::serialization::store_t_to_binary(arg, fullBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, std::move(fullBlob), std::move(fullConnections)); } return true; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 79c2edf1d..57b1d049c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -30,8 +30,8 @@ #pragma once -#include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "cryptonote_protocol/enums.h" #include "cryptonote_basic/connection_context.h" namespace cryptonote { diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 1e9f3e399..0b065c3c3 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -159,7 +159,7 @@ namespace levin return get_out_connections(p2p, get_blockchain_height(p2p, core)); } - epee::byte_slice make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) + epee::levin::message_writer make_tx_message(std::vector<blobdata>&& txs, const bool pad, const bool fluff) { NOTIFY_NEW_TRANSACTIONS::request request{}; request.txs = std::move(txs); @@ -193,21 +193,17 @@ namespace levin // if the size of _ moved enough, we might lose byte in size encoding, we don't care } - epee::byte_slice fullBlob; - if (!epee::serialization::store_t_to_binary(request, fullBlob)) + epee::levin::message_writer out; + if (!epee::serialization::store_t_to_binary(request, out.buffer)) throw std::runtime_error{"Failed to serialize to epee binary format"}; - return fullBlob; + return out; } bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff) { - const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); - p2p.for_connection(destination, [&blob](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, blob.size(), NOTIFY_NEW_TRANSACTIONS::ID); - return true; - }); - return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); + epee::byte_slice blob = make_tx_message(std::move(txs), pad, fluff).finalize_notify(NOTIFY_NEW_TRANSACTIONS::ID); + return p2p.send(std::move(blob), destination); } /* The current design uses `asio::strand`s. The documentation isn't as clear @@ -653,10 +649,6 @@ namespace levin else message = zone_->noise.clone(); - zone_->p2p->for_connection(channel.connection, [&](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, message.size(), "noise"); - return true; - }); if (zone_->p2p->send(std::move(message), channel.connection)) { if (!channel.queue.empty() && channel.active.empty()) @@ -816,9 +808,8 @@ namespace levin // Padding is not useful when using noise mode. Send as stem so receiver // forwards in Dandelion++ mode. - const epee::byte_slice payload = make_tx_payload(std::move(txs), false, false); epee::byte_slice message = epee::levin::make_fragmented_notify( - zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(payload) + zone_->noise.size(), NOTIFY_NEW_TRANSACTIONS::ID, make_tx_message(std::move(txs), false, false) ); if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size()) { diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index be38752da..14233bf29 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -30,6 +30,8 @@ #include "common/command_line.h" #include "net/parse.h" #include "daemon/command_parser_executor.h" +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 7c3e8785d..c2070b0d1 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -28,6 +28,8 @@ // #include "device_trezor.hpp" +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> namespace hw { namespace trezor { diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 70dc7f539..5f21ecd49 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -31,6 +31,7 @@ #include "memwipe.h" #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/split.hpp> +#include <boost/algorithm/string/predicate.hpp> #include <boost/regex.hpp> namespace hw { diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index a56090de0..be95868d0 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -38,6 +38,7 @@ #include <boost/asio/ip/udp.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> #include <boost/format.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "common/apply_permutation.h" #include "transport.hpp" #include "messages/messages-common.pb.h" diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp index 6c03b3808..ada4eb0d3 100644 --- a/src/net/i2p_address.cpp +++ b/src/net/i2p_address.cpp @@ -38,7 +38,7 @@ #include "net/error.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" -#include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/parse.cpp b/src/net/parse.cpp index 8a98e941a..298576ba4 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -31,6 +31,7 @@ #include "net/tor_address.h" #include "net/i2p_address.h" #include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp index 9db9d4483..c797a24d4 100644 --- a/src/net/socks_connect.cpp +++ b/src/net/socks_connect.cpp @@ -38,6 +38,7 @@ #include "net/net_utils_base.h" #include "net/socks.h" #include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp index 4414861e7..a04dcb042 100644 --- a/src/net/tor_address.cpp +++ b/src/net/tor_address.cpp @@ -38,7 +38,7 @@ #include "net/error.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" -#include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index db931122e..f2888674b 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -344,10 +344,9 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) final; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay); - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) final; virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 4571743a2..a0b8438b2 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -682,11 +682,13 @@ namespace nodetool full_addrs.insert("212.83.175.67:28080"); full_addrs.insert("212.83.172.165:28080"); full_addrs.insert("192.110.160.146:28080"); + full_addrs.insert("88.99.173.38:28080"); } else if (m_nettype == cryptonote::STAGENET) { full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("192.110.160.146:38080"); + full_addrs.insert("88.99.173.38:38080"); } else if (m_nettype == cryptonote::FAKECHAIN) { @@ -701,6 +703,7 @@ namespace nodetool full_addrs.insert("209.250.243.248:18080"); full_addrs.insert("104.238.221.81:18080"); full_addrs.insert("66.85.74.134:18080"); + full_addrs.insert("88.99.173.38:18080"); } return full_addrs; } @@ -1229,8 +1232,8 @@ namespace nodetool if(!max_index) return 0; - size_t x = crypto::rand<size_t>()%(max_index+1); - size_t res = (x*x*x)/(max_index*max_index); //parabola \/ + size_t x = crypto::rand<size_t>()%(16*max_index+1); + size_t res = (x*x*x)/(max_index*max_index*16*16*16); //parabola \/ MDEBUG("Random connection index=" << res << "(x="<< x << ", max_index=" << max_index << ")"); return res; } @@ -2175,8 +2178,9 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, epee::levin::message_writer data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { + epee::byte_slice message = data_buff.finalize_notify(command); std::sort(connections.begin(), connections.end()); auto zone = m_network_zones.begin(); for(const auto& c_id: connections) @@ -2194,7 +2198,7 @@ namespace nodetool ++zone; } if (zone->first == c_id.first) - zone->second.m_net_server.get_config_object().notify(command, data_buff, c_id.second); + zone->second.m_net_server.get_config_object().send(message.clone(), c_id.second); } return true; } @@ -2261,24 +2265,13 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(const int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { if(is_filtered_command(context.m_remote_address, command)) return false; network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); - return res > 0; - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - if(is_filtered_command(context.m_remote_address, command)) - return false; - - network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); + int res = zone.m_net_server.get_config_object().send(message.finalize_notify(command), context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 0da758ad4..92b7596ae 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -40,6 +40,8 @@ #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" +namespace epee { namespace levin { class message_writer; } } + namespace nodetool { @@ -49,10 +51,9 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay)=0; - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_public_connections_count()=0; @@ -71,7 +72,7 @@ namespace nodetool template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { return false; } @@ -79,11 +80,7 @@ namespace nodetool { return epee::net_utils::zone::invalid; } - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - return false; - } - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { return true; } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 40c04b4f9..ded545efa 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -30,6 +30,7 @@ #include <boost/preprocessor/stringize.hpp> #include <boost/uuid/nil_generator.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -604,7 +605,7 @@ namespace cryptonote if (max_blocks == 0) { res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; - return false; + return true; } } @@ -613,7 +614,7 @@ namespace cryptonote { res.status = "Failed"; add_host_fail(ctx); - return false; + return true; } CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); @@ -649,12 +650,12 @@ namespace cryptonote if (!r) { res.status = "Failed"; - return false; + return true; } if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) { res.status = "Failed"; - return false; + return true; } for (size_t i = 0; i < indices.size(); ++i) res.output_indices.back().indices.push_back({std::move(indices[i])}); @@ -677,7 +678,7 @@ namespace cryptonote if(!m_core.get_alternative_blocks(blks)) { res.status = "Failed"; - return false; + return true; } res.blks_hashes.reserve(blks.size()); @@ -748,7 +749,7 @@ namespace cryptonote { res.status = "Failed"; add_host_fail(ctx); - return false; + return true; } CHECK_PAYMENT_SAME_TS(req, res, res.m_block_ids.size() * COST_PER_BLOCK_HASH); @@ -1079,7 +1080,7 @@ namespace cryptonote if (!r) { res.status = "Failed"; - return false; + return true; } } } @@ -1455,7 +1456,8 @@ namespace cryptonote res.status = peer_list_res.status; if (!success) { - return false; + res.status = "Failed to get peer list"; + return true; } if (res.status != CORE_RPC_STATUS_OK) { @@ -1657,7 +1659,7 @@ namespace cryptonote if (m_should_use_bootstrap_daemon) { res.status = "This command is unsupported for bootstrap daemon"; - return false; + return true; } } res.count = m_core.get_current_blockchain_height(); @@ -1673,7 +1675,7 @@ namespace cryptonote if (m_should_use_bootstrap_daemon) { res = "This command is unsupported for bootstrap daemon"; - return false; + return true; } } if(req.size() != 1) @@ -1983,7 +1985,8 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); if (m_should_use_bootstrap_daemon) { - res.status = "This command is unsupported for bootstrap daemon"; + error_resp.code = CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP; + error_resp.message = "This command is unsupported for bootstrap daemon"; return false; } } @@ -2675,7 +2678,7 @@ namespace cryptonote if (!m_core.get_blockchain_storage().flush_txes_from_pool(txids)) { res.status = "Failed to remove one or more tx(es)"; - return false; + return true; } if (failed) @@ -2684,7 +2687,7 @@ namespace cryptonote res.status = "Failed to parse txid"; else res.status = "Failed to parse some of the txids"; - return false; + return true; } res.status = CORE_RPC_STATUS_OK; @@ -2843,7 +2846,7 @@ namespace cryptonote if (req.limit_down != -1) { res.status = CORE_RPC_ERROR_CODE_WRONG_PARAM; - return false; + return true; } epee::net_utils::connection_basic::set_rate_down_limit(nodetool::default_limit_down); } @@ -2857,7 +2860,7 @@ namespace cryptonote if (req.limit_up != -1) { res.status = CORE_RPC_ERROR_CODE_WRONG_PARAM; - return false; + return true; } epee::net_utils::connection_basic::set_rate_up_limit(nodetool::default_limit_up); } @@ -2961,17 +2964,17 @@ namespace cryptonote if (!tools::download(path.string(), res.auto_uri)) { MERROR("Failed to download " << res.auto_uri); - return false; + return true; } if (!tools::sha256sum(path.string(), file_hash)) { MERROR("Failed to hash " << path); - return false; + return true; } if (hash != epee::string_tools::pod_to_hex(file_hash)) { MERROR("Download from " << res.auto_uri << " does not match the expected hash"); - return false; + return true; } MINFO("New version downloaded to " << path); } @@ -3046,6 +3049,8 @@ namespace cryptonote if (failed) { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = res.status; return false; } @@ -3157,7 +3162,7 @@ namespace cryptonote if (!req.binary) { res.status = "Binary only call"; - return false; + return true; } try { @@ -3169,7 +3174,7 @@ namespace cryptonote if (!data) { res.status = "Failed to get output distribution"; - return false; + return true; } res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); @@ -3178,7 +3183,7 @@ namespace cryptonote catch (const std::exception &e) { res.status = "Failed to get output distribution"; - return false; + return true; } res.status = CORE_RPC_STATUS_OK; @@ -3412,7 +3417,8 @@ namespace cryptonote if (!m_rpc_payment) { - res.status = "Payments not enabled"; + error_resp.code = CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED; + error_resp.message = "Payments not enabled"; return false; } @@ -3440,7 +3446,8 @@ namespace cryptonote if (!m_rpc_payment) { - res.status = "Payments not enabled"; + error_resp.code = CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED; + error_resp.message = "Payments not enabled"; return false; } diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 1458049ab..232df0373 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -49,6 +49,8 @@ #define CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT -17 #define CORE_RPC_ERROR_CODE_STALE_PAYMENT -18 #define CORE_RPC_ERROR_CODE_RESTRICTED -19 +#define CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP -20 +#define CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED -21 static inline const char *get_rpc_server_error_message(int64_t code) { @@ -72,6 +74,8 @@ static inline const char *get_rpc_server_error_message(int64_t code) case CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT: return "Duplicate payment"; case CORE_RPC_ERROR_CODE_STALE_PAYMENT: return "Stale payment"; case CORE_RPC_ERROR_CODE_RESTRICTED: return "Parameters beyond restricted allowance"; + case CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP: return "Command is unsupported in bootstrap mode"; + case CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED: return "Payments not enabled"; default: MERROR("Unknown error: " << code); return "Unknown error"; } } diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 176f11fa3..bf6584f72 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/filesystem.hpp> #include "cryptonote_config.h" #include "include_base_utils.h" #include "string_tools.h" diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 2ae091c7e..a7856d60f 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -50,6 +50,7 @@ #include <boost/format.hpp> #include <boost/regex.hpp> #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "console_handler.h" #include "common/i18n.h" @@ -607,9 +608,14 @@ void simple_wallet::handle_transfer_exception(const std::exception_ptr &e, bool fail_msg_writer() << e.what(); warn_of_possible_attack = false; } + catch (const tools::error::zero_amount&) + { + fail_msg_writer() << sw::tr("destination amount is zero"); + warn_of_possible_attack = false; + } catch (const tools::error::zero_destination&) { - fail_msg_writer() << sw::tr("one of destinations is zero"); + fail_msg_writer() << sw::tr("transaction has no destination"); warn_of_possible_attack = false; } catch (const tools::error::tx_too_big& e) diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index 24f6d37db..b28ffd64c 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -40,6 +40,7 @@ #include <vector> #include <sstream> #include <boost/format.hpp> +#include <boost/filesystem.hpp> using namespace std; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 9d3ec4399..adff042ad 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -45,10 +45,8 @@ #include <sstream> #include <unordered_map> -#ifdef WIN32 #include <boost/locale.hpp> #include <boost/filesystem.hpp> -#endif using namespace std; using namespace cryptonote; @@ -1567,8 +1565,10 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri setStatusError(writer.str()); } catch (const tools::error::tx_sum_overflow& e) { setStatusError(e.what()); + } catch (const tools::error::zero_amount&) { + setStatusError(tr("destination amount is zero")); } catch (const tools::error::zero_destination&) { - setStatusError(tr("one of destinations is zero")); + setStatusError(tr("transaction has no destination")); } catch (const tools::error::tx_too_big& e) { setStatusError(tr("failed to find a suitable way to split transactions")); } catch (const tools::error::transfer_error& e) { @@ -2310,6 +2310,10 @@ bool WalletImpl::rescanSpent() return true; } +void WalletImpl::setOffline(bool offline) +{ + m_wallet->set_offline(offline); +} void WalletImpl::hardForkInfo(uint8_t &version, uint64_t &earliest_height) const { diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 0f3614bb4..13b33d1cd 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -182,6 +182,8 @@ public: virtual bool setCacheAttribute(const std::string &key, const std::string &val) override; virtual std::string getCacheAttribute(const std::string &key) const override; + virtual void setOffline(bool offline) override; + virtual bool setUserNote(const std::string &txid, const std::string ¬e) override; virtual std::string getUserNote(const std::string &txid) const override; virtual std::string getTxKey(const std::string &txid) const override; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index b1cebedaf..320b458bd 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -1014,6 +1014,12 @@ struct Wallet * \return true on success */ virtual bool rescanSpent() = 0; + + /* + * \brief setOffline - toggle set offline on/off + * \param offline - true/false + */ + virtual void setOffline(bool offline) = 0; //! blackballs a set of outputs virtual bool blackballOutputs(const std::vector<std::string> &outputs, bool add) = 0; diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 87cb75fbf..b7b29420b 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -30,6 +30,8 @@ #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/format.hpp> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <fstream> #include <sstream> #include "file_io_utils.h" diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 0af896c76..c3325c24d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -37,6 +37,7 @@ #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/algorithm/string/predicate.hpp> #include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/preprocessor/stringize.hpp> @@ -8760,7 +8761,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent // throw if total amount overflows uint64_t for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); @@ -8919,7 +8920,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry // throw if total amount overflows uint64_t for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); @@ -9892,14 +9893,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp needed_money = 0; for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, 0, m_nettype); } // throw if attempting a transaction with no money - THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_amount); std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false); diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 4a89ed81a..011780f43 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -83,6 +83,7 @@ namespace tools // tx_rejected // tx_sum_overflow // tx_too_big + // zero_amount // zero_destination // wallet_rpc_error * // daemon_busy @@ -750,10 +751,18 @@ namespace tools uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- + struct zero_amount: public transfer_error + { + explicit zero_amount(std::string&& loc) + : transfer_error(std::move(loc), "destination amount is zero") + { + } + }; + //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error { explicit zero_destination(std::string&& loc) - : transfer_error(std::move(loc), "destination amount is zero") + : transfer_error(std::move(loc), "transaction has no destination") { } }; diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index b39a40b64..0b200ae60 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -822,7 +822,7 @@ namespace tools if (at_least_one_destination && dsts.empty()) { er.code = WALLET_RPC_ERROR_CODE_ZERO_DESTINATION; - er.message = "No destinations for this transfer"; + er.message = "Transaction has no destination"; return false; } @@ -3359,6 +3359,11 @@ namespace tools er.code = WALLET_RPC_ERROR_CODE_DAEMON_IS_BUSY; er.message = e.what(); } + catch (const tools::error::zero_amount& e) + { + er.code = WALLET_RPC_ERROR_CODE_ZERO_AMOUNT; + er.message = e.what(); + } catch (const tools::error::zero_destination& e) { er.code = WALLET_RPC_ERROR_CODE_ZERO_DESTINATION; diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index f7c5bb0e1..b991029a9 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -76,4 +76,5 @@ #define WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC -43 #define WALLET_RPC_ERROR_CODE_INVALID_LOG_LEVEL -44 #define WALLET_RPC_ERROR_CODE_ATTRIBUTE_NOT_FOUND -45 +#define WALLET_RPC_ERROR_CODE_ZERO_AMOUNT -46 #define WALLET_RPC_ERROR_CODE_INVALID_SIGNATURE_TYPE -47 diff --git a/tests/functional_tests/make_test_signature.cc b/tests/functional_tests/make_test_signature.cc index 07cc3eea6..cb1723847 100644 --- a/tests/functional_tests/make_test_signature.cc +++ b/tests/functional_tests/make_test_signature.cc @@ -28,6 +28,7 @@ #include <stdio.h> #include "misc_language.h" +#include "misc_log_ex.h" #include "string_tools.h" #include "rpc/rpc_payment_signature.h" diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index 78b7b6863..209a75221 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -68,13 +68,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf.clone(); + buff_out.write(epee::to_span(m_invoke_out_buf)); return m_return_code; } diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index baab07d31..59eef7bd1 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -67,7 +67,7 @@ namespace net_load_tests { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_connection_context& context) { //m_invoke_counter.inc(); //std::unique_lock<std::mutex> lock(m_mutex); diff --git a/tests/unit_tests/epee_boosted_tcp_server.cpp b/tests/unit_tests/epee_boosted_tcp_server.cpp index 457c05c15..84fc0a29b 100644 --- a/tests/unit_tests/epee_boosted_tcp_server.cpp +++ b/tests/unit_tests/epee_boosted_tcp_server.cpp @@ -153,7 +153,7 @@ TEST(test_epee_connection, test_lifetime) delay(delay), on_connection_close_f(on_connection_close_f) {} - virtual int invoke(int, const epee::span<const uint8_t>, epee::byte_slice&, context_t&) override { epee::misc_utils::sleep_no_w(delay); return {}; } + virtual int invoke(int, const epee::span<const uint8_t>, epee::byte_stream&, context_t&) override { epee::misc_utils::sleep_no_w(delay); return {}; } virtual int notify(int, const epee::span<const uint8_t>, context_t&) override { return {}; } virtual void callback(context_t&) override {} virtual void on_connection_new(context_t&) override {} @@ -282,7 +282,7 @@ TEST(test_epee_connection, test_lifetime) for (auto i = 0; i < N; ++i) { tag = create_connection(); ASSERT_TRUE(shared_state->get_connections_count() == 1); - success = shared_state->invoke_async(1, {}, tag, [](int, const epee::span<const uint8_t>, context_t&){}, TIMEOUT); + success = shared_state->invoke_async(1, epee::levin::message_writer{}, tag, [](int, const epee::span<const uint8_t>, context_t&){}, TIMEOUT); ASSERT_TRUE(success); while (shared_state->sock_count == 1) { success = shared_state->foreach_connection([&shared_state, &tag](context_t&){ diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index a499fa608..d9de99b8b 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -59,13 +59,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf.clone(); + buff_out.write(epee::to_span(m_invoke_out_buf)); return m_return_code; } @@ -434,8 +434,11 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process const int expected_command = 4673261; const std::string in_data(256, 'e'); + epee::levin::message_writer message{}; + message.buffer.write(epee::to_span(in_data)); + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); + const epee::byte_slice notify = message.finalize_notify(expected_command); test_connection_ptr conn = create_connection(); @@ -468,11 +471,16 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process const int expected_command = 4673261; const int expected_fragmented_command = 46732; const std::string in_data(256, 'e'); - std::string in_fragmented_data(1024 * 4, 'c'); + + epee::levin::message_writer message{}; + message.buffer.write(epee::to_span(in_data)); + + epee::levin::message_writer in_fragmented_data; + in_fragmented_data.buffer.put_n('c', 1024 * 4); const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); - epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise, expected_fragmented_command, epee::strspan<std::uint8_t>(in_fragmented_data)); + const epee::byte_slice notify = message.finalize_notify(expected_command); + epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise.size(), expected_fragmented_command, std::move(in_fragmented_data)); EXPECT_EQ(5u, fragmented.size() / 1024); EXPECT_EQ(0u, fragmented.size() % 1024); @@ -497,11 +505,13 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process ASSERT_TRUE(conn->m_protocol_handler.handle_recv(next.data(), next.size())); } - in_fragmented_data.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + std::string compare_buffer(1024 * 4, 'c'); + compare_buffer.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + ASSERT_EQ(4u, m_commands_handler.notify_counter()); ASSERT_EQ(0u, m_commands_handler.invoke_counter()); ASSERT_EQ(expected_fragmented_command, m_commands_handler.last_command()); - ASSERT_EQ(in_fragmented_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(compare_buffer, m_commands_handler.last_in_buf()); ASSERT_EQ(0u, conn->send_counter()); ASSERT_TRUE(conn->last_send_data().empty()); diff --git a/tests/unit_tests/get_xtype_from_string.cpp b/tests/unit_tests/get_xtype_from_string.cpp index 78bbe6657..17492ed19 100644 --- a/tests/unit_tests/get_xtype_from_string.cpp +++ b/tests/unit_tests/get_xtype_from_string.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include <string_tools.h> +#include <string_tools_lexical.h> using namespace epee::string_tools; diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index eee9e224d..30d6f8133 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -245,9 +245,9 @@ namespace return out; } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, cryptonote::levin::detail::p2p_context& context) override final + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, cryptonote::levin::detail::p2p_context& context) override final { - buff_out = nullptr; + buff_out.clear(); invoked_.push_back( {context.m_connection_id, command, std::string{reinterpret_cast<const char*>(in_buff.data()), in_buff.size()}} ); @@ -384,21 +384,50 @@ TEST(make_header, expect_return) EXPECT_EQ(0u, header1.m_flags); } -TEST(make_notify, empty_payload) +TEST(message_writer, invoke_with_empty_payload) { - const epee::byte_slice message = epee::levin::make_notify(443, nullptr); + const epee::byte_slice message = epee::levin::message_writer{}.finalize_invoke(443); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, true); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(message_writer, invoke_with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_invoke(443); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, true); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(message_writer, notify_with_empty_payload) +{ + const epee::byte_slice message = epee::levin::message_writer{}.finalize_notify(443); const epee::levin::bucket_head2 header = epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, false); ASSERT_EQ(sizeof(header), message.size()); EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); } -TEST(make_notify, with_payload) +TEST(message_writer, notify_with_payload) { std::string bytes(100, 'a'); std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); - const epee::byte_slice message = epee::levin::make_notify(443, epee::strspan<std::uint8_t>(bytes)); + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_notify(443); const epee::levin::bucket_head2 header = epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, false); @@ -407,6 +436,44 @@ TEST(make_notify, with_payload) EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); } +TEST(message_writer, response_with_empty_payload) +{ + const epee::byte_slice message = epee::levin::message_writer{}.finalize_response(443, 1); + epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_RESPONSE, false); + header.m_return_code = SWAP32LE(1); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(message_writer, response_with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_response(443, 6450); + epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_RESPONSE, false); + header.m_return_code = SWAP32LE(6450); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(message_writer, error) +{ + epee::levin::message_writer writer{}; + writer.buffer.clear(); + + EXPECT_THROW(writer.finalize_invoke(0), std::runtime_error); + EXPECT_THROW(writer.finalize_notify(0), std::runtime_error); + EXPECT_THROW(writer.finalize_response(0, 0), std::runtime_error); +} + TEST(make_noise, invalid) { EXPECT_TRUE(epee::levin::make_noise_notify(sizeof(epee::levin::bucket_head2) - 1).empty()); @@ -428,13 +495,13 @@ TEST(make_noise, valid) TEST(make_fragment, invalid) { - EXPECT_TRUE(epee::levin::make_fragmented_notify(nullptr, 0, nullptr).empty()); + EXPECT_TRUE(epee::levin::make_fragmented_notify(0, 0, epee::levin::message_writer{}).empty()); } TEST(make_fragment, single) { const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 11, nullptr); + const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise.size(), 11, epee::levin::message_writer{}); const epee::levin::bucket_head2 header = epee::levin::make_header(11, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_REQUEST, false); @@ -449,8 +516,13 @@ TEST(make_fragment, multiple) std::string bytes(1024 * 3 - 150, 'a'); std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + epee::levin::message_writer message; + message.buffer.write(epee::to_span(bytes)); + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 114, epee::strspan<std::uint8_t>(bytes)); + epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise.size(), 114, std::move(message)); + + EXPECT_EQ(1024 * 3, fragment.size()); epee::levin::bucket_head2 header = epee::levin::make_header(0, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_BEGIN, false); @@ -497,6 +569,7 @@ TEST(make_fragment, multiple) fragment.take_slice(bytes.size()); + EXPECT_EQ(18, fragment.size()); EXPECT_EQ(18, std::count(fragment.cbegin(), fragment.cend(), 0)); } @@ -2164,20 +2237,31 @@ TEST_F(levin_notify, command_max_bytes) add_connection(true); - std::string bytes(4096, 'h'); + std::string payload(4096, 'h'); + epee::byte_slice bytes; + { + epee::levin::message_writer dest{}; + dest.buffer.write(epee::to_span(payload)); + bytes = dest.finalize_notify(ping_command); + } - EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1, get_connections().send(bytes.clone(), contexts_.front().get_id())); EXPECT_EQ(1u, contexts_.front().process_send_queue(true)); EXPECT_EQ(1u, receiver_.notified_size()); const received_message msg = receiver_.get_raw_notification(); EXPECT_EQ(ping_command, msg.command); EXPECT_EQ(contexts_.front().get_id(), msg.connection); - EXPECT_EQ(bytes, msg.payload); + EXPECT_EQ(payload, msg.payload); - bytes.push_back('e'); + { + payload.push_back('h'); + epee::levin::message_writer dest{}; + dest.buffer.write(epee::to_span(payload)); + bytes = dest.finalize_notify(ping_command); + } - EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1, get_connections().send(std::move(bytes), contexts_.front().get_id())); EXPECT_EQ(1u, contexts_.front().process_send_queue(false)); EXPECT_EQ(0u, receiver_.notified_size()); } diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index 8d5c784eb..2c80acda5 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -351,7 +351,7 @@ TEST(cryptonote_protocol_handler, race_condition) acceptor.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); acceptor.bind(endpoint, ec); EXPECT_EQ(ec.value(), 0); - acceptor.listen(boost::asio::socket_base::max_listen_connections, ec); + acceptor.listen(boost::asio::socket_base::max_connections, ec); EXPECT_EQ(ec.value(), 0); out->socket().open(endpoint.protocol(), ec); EXPECT_EQ(ec.value(), 0); @@ -449,7 +449,6 @@ TEST(cryptonote_protocol_handler, race_condition) }; struct net_node_t: commands_handler_t, p2p_endpoint_t { using span_t = epee::span<const uint8_t>; - using string_t = std::string; using zone_t = epee::net_utils::zone; using uuid_t = boost::uuids::uuid; using relay_t = cryptonote::relay_method; @@ -462,12 +461,9 @@ TEST(cryptonote_protocol_handler, race_condition) using subnets = std::map<epee::net_utils::ipv4_network_subnet, time_t>; using hosts = std::map<std::string, time_t>; }; - struct slice { - using bytes = epee::byte_slice; - }; shared_state_ptr shared_state; core_protocol_ptr core_protocol; - virtual int invoke(int command, const span_t in, slice::bytes &out, context_t &context) override { + virtual int invoke(int command, const span_t in, epee::byte_stream &out, context_t &context) override { if (core_protocol) { if (command == messages::handshake::ID) { return epee::net_utils::buff_to_t_adapter<void, typename messages::handshake::request, typename messages::handshake::response>( @@ -491,7 +487,7 @@ TEST(cryptonote_protocol_handler, race_condition) virtual int notify(int command, const span_t in, context_t &context) override { if (core_protocol) { bool handled; - slice::bytes out; + epee::byte_stream out; return core_protocol->handle_invoke_map(true, command, in, out, context, handled); } else @@ -527,22 +523,16 @@ TEST(cryptonote_protocol_handler, race_condition) else return {}; } - virtual bool invoke_command_to_peer(int command, const span_t in, string_t& out, const contexts::basic& context) override { - if (shared_state) - return shared_state->invoke(command, in, out, context.m_connection_id); - else - return {}; - } - virtual bool invoke_notify_to_peer(int command, const span_t in, const contexts::basic& context) override { + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer in, const contexts::basic& context) override { if (shared_state) - return shared_state->notify(command, in, context.m_connection_id); + return shared_state->send(in.finalize_notify(command), context.m_connection_id); else return {}; } - virtual bool relay_notify_to_list(int command, const span_t in, connections_t connections) override { + virtual bool relay_notify_to_list(int command, epee::levin::message_writer in, connections_t connections) override { if (shared_state) { for (auto &e: connections) - shared_state->notify(command, in, e.second); + shared_state->send(in.finalize_notify(command), e.second); } return {}; } |