diff options
Diffstat (limited to 'contrib')
27 files changed, 839 insertions, 623 deletions
diff --git a/contrib/brew/Brewfile b/contrib/brew/Brewfile index 1fdf45cfe..36b073b25 100644 --- a/contrib/brew/Brewfile +++ b/contrib/brew/Brewfile @@ -27,6 +27,7 @@ brew "miniupnpc" brew "readline" brew "ldns" brew "expat" +brew "ccache" brew "doxygen" brew "graphviz" brew "libunwind-headers" diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 62e975e50..d80775b22 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=1.1.1i +$(package)_version=1.1.1k $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 +$(package)_sha256_hash=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h index 6b79f6d92..18d60e088 100644 --- a/contrib/epee/include/byte_slice.h +++ b/contrib/epee/include/byte_slice.h @@ -112,7 +112,7 @@ namespace epee explicit byte_slice(std::string&& buffer); //! Convert `stream` into a slice with zero allocations. - explicit byte_slice(byte_stream&& stream) noexcept; + explicit byte_slice(byte_stream&& stream, bool shrink = true); byte_slice(byte_slice&& source) noexcept; ~byte_slice() noexcept = default; diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 25f8c648b..84dc79266 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -24,211 +24,23 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include <fstream> -#include <boost/filesystem/path.hpp> -#include <boost/filesystem/operations.hpp> -#ifdef WIN32 -#include <windows.h> -#include "string_tools.h" -#endif - -// On Windows there is a problem with non-ASCII characters in path and file names -// as far as support by the standard components used is concerned: - -// The various file stream classes, e.g. std::ifstream and std::ofstream, are -// part of the GNU C++ Library / libstdc++. On the most basic level they use the -// fopen() call as defined / made accessible to programs compiled within MSYS2 -// by the stdio.h header file maintained by the MinGW project. - -// The critical point: The implementation of fopen() is part of MSVCRT, the -// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any -// Unicode support. - -// Monero code that would want to continue to use the normal file stream classes -// but WITH Unicode support could therefore not solve this problem on its own, -// but 2 different projects from 2 different maintaining groups would need changes -// in this particular direction - something probably difficult to achieve and -// with a long time to wait until all new versions / releases arrive. - -// Implemented solution approach: Circumvent the problem by stopping to use std -// file stream classes on Windows and directly use Unicode-capable WIN32 API -// calls. Most of the code doing so is concentrated in this header file here. +#include <string> +#include <ctime> namespace epee { namespace file_io_utils { - inline - bool is_file_exist(const std::string& path) - { - boost::filesystem::path p(path); - return boost::filesystem::exists(p); - } - - inline - bool save_string_to_file(const std::string& path_to_file, const std::string& str) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD bytes_written; - DWORD bytes_to_write = (DWORD)str.size(); - BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); - CloseHandle(file_handle); - if (bytes_written != bytes_to_write) - result = FALSE; - return result; -#else - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool get_file_time(const std::string& path_to_file, time_t& ft) - { - boost::system::error_code ec; - ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); - if(!ec) - return true; - else - return false; - } - - inline - bool set_file_time(const std::string& path_to_file, const time_t& ft) - { - boost::system::error_code ec; - boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); - if(!ec) - return true; - else - return false; - } - - - inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { - CloseHandle(file_handle); - return false; - } - target_str.resize(file_size); - DWORD bytes_read; - BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); - CloseHandle(file_handle); - if (bytes_read != file_size) - result = FALSE; - return result; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - - std::ifstream::pos_type file_size = fstream.tellg(); - - if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large - return false;//don't go crazy - size_t file_size_t = static_cast<size_t>(file_size); - - target_str.resize(file_size_t); - - fstream.seekg (0, std::ios::beg); - fstream.read((char*)target_str.data(), target_str.size()); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool append_string_to_file(const std::string& path_to_file, const std::string& str) - { - // No special Windows implementation because so far not used in Monero code - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } - } - - inline - bool get_file_size(const std::string& path_to_file, uint64_t &size) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER file_size; - BOOL result = GetFileSizeEx(file_handle, &file_size); - CloseHandle(file_handle); - if (result) { - size = file_size.QuadPart; - } - return size; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - size = fstream.tellg(); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - + bool is_file_exist(const std::string& path); + bool save_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_time(const std::string& path_to_file, time_t& ft); + bool set_file_time(const std::string& path_to_file, const time_t& ft); + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000); + bool append_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_size(const std::string& path_to_file, uint64_t &size); } } diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index a04c63231..5ccfe6fcc 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -28,9 +28,11 @@ #pragma once -#include <limits> -#include <boost/thread.hpp> #include <boost/utility/value_init.hpp> +#include <boost/shared_ptr.hpp> +#include <limits> +#include <functional> +#include <vector> namespace epee { #define STD_TRY_BEGIN() try { @@ -95,16 +97,7 @@ namespace misc_utils return memcmp(&_Left, &_Right, sizeof(_Left)) < 0; } - - inline - bool sleep_no_w(long ms ) - { - boost::this_thread::sleep( - boost::get_system_time() + - boost::posix_time::milliseconds( std::max<long>(ms,0) ) ); - - return true; - } + bool sleep_no_w(long ms ); template <typename T> T get_mid(const T &a, const T &b) diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 3be335e85..392ec11eb 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -41,7 +41,7 @@ #define MAX_LOG_FILES 50 #define MCLOG_TYPE(level, cat, color, type, x) do { \ - if (ELPP->vRegistry()->allowed(level, cat)) { \ + if (el::Loggers::allowed(level, cat)) { \ el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ } while (0) @@ -89,7 +89,7 @@ #define IFLOG(level, cat, color, type, init, x) \ do { \ - if (ELPP->vRegistry()->allowed(level, cat)) { \ + if (el::Loggers::allowed(level, cat)) { \ init; \ el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 3c31cf22b..f40cd108a 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -265,6 +265,12 @@ namespace net_utils template<class t_callback> bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); + boost::asio::ssl::context& get_ssl_context() noexcept + { + assert(m_state != nullptr); + return m_state->ssl_context; + } + typename t_protocol_handler::config_type& get_config_object() { assert(m_state != nullptr); // always set in constructor diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index cb1388f3b..61e2b30fe 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -40,6 +40,7 @@ #include <boost/date_time/posix_time/posix_time.hpp> // TODO #include <boost/thread/condition_variable.hpp> // TODO #include <boost/make_shared.hpp> +#include <boost/thread.hpp> #include "warnings.h" #include "string_tools.h" #include "misc_language.h" @@ -269,8 +270,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) //_dbg3("[sock " << socket().native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); CRITICAL_REGION_LOCAL(self->m_self_refs_lock); //_dbg3("[sock " << socket().native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); - if(m_was_shutdown) - return false; ++m_reference_count; m_self_ref = std::move(self); return true; @@ -562,7 +561,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { // LOCK: chunking epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** - MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<(const void*)message_data); // 01234567890 // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 @@ -575,14 +574,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) while (!message.empty()) { byte_slice chunk = message.take_slice(chunksize_good); - MDEBUG("chunk_start="<<(void*)chunk.data()<<" ptr="<<message_data<<" pos="<<(chunk.data() - message_data)); + MDEBUG("chunk_start="<<(void*)chunk.data()<<" ptr="<<(const void*)message_data<<" pos="<<(chunk.data() - message_data)); MDEBUG("part of " << message.size() << ": pos="<<(chunk.data() - message_data) << " len="<<chunk.size()); bool ok = do_send_chunk(std::move(chunk)); // <====== *** all_ok = all_ok && ok; if (!all_ok) { - MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<(const void*)message_data); MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless " << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << message_size); return false; // partial failure in sending @@ -590,7 +589,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) // (in catch block, or uniq pointer) delete buf; } // each chunk - MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<(const void*)message_data); MDEBUG("do_send() m_connection_type = " << m_connection_type); diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 1665fdac7..ffb3f3b7e 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -74,7 +74,13 @@ uint64_t ticks = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::request> req; \ bool parse_res = epee::serialization::load_t_from_json(static_cast<command_type::request&>(req), query_info.m_body); \ - CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse json: \r\n" << query_info.m_body); \ + if (!parse_res) \ + { \ + MERROR("Failed to parse json: \r\n" << query_info.m_body); \ + response_info.m_response_code = 400; \ + response_info.m_response_comment = "Bad request"; \ + return true; \ + } \ uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ @@ -104,7 +110,13 @@ uint64_t ticks = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::request> req; \ bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), epee::strspan<uint8_t>(query_info.m_body)); \ - CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \ + if (!parse_res) \ + { \ + MERROR("Failed to parse bin body data, body size=" << query_info.m_body.size()); \ + response_info.m_response_code = 400; \ + response_info.m_response_comment = "Bad request"; \ + return true; \ + } \ uint64_t ticks1 = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index fce6d4b7e..df59a6c44 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,6 +31,7 @@ #include <cstdint> +#include "byte_stream.h" #include "net_utils_base.h" #include "span.h" @@ -83,11 +84,12 @@ namespace levin #define LEVIN_PROTOCOL_VER_0 0 #define LEVIN_PROTOCOL_VER_1 1 + template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; @@ -125,12 +127,41 @@ namespace levin } } + //! Provides space for levin (p2p) header, so that payload can be sent without copy + class message_writer + { + byte_slice finalize(uint32_t command, uint32_t flags, uint32_t return_code, bool expect_response); + public: + using header = bucket_head2; + + explicit message_writer(std::size_t reserve = 8192); + + message_writer(const message_writer&) = delete; + message_writer(message_writer&&) = default; + ~message_writer() = default; + message_writer& operator=(const message_writer&) = delete; + message_writer& operator=(message_writer&&) = default; + + //! \return Size of payload (excludes header size). + std::size_t payload_size() const noexcept + { + return buffer.size() < sizeof(header) ? 0 : buffer.size() - sizeof(header); + } + + byte_slice finalize_invoke(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, true); } + byte_slice finalize_notify(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, false); } + byte_slice finalize_response(uint32_t command, uint32_t return_code) + { + return finalize(command, LEVIN_PACKET_RESPONSE, return_code, false); + } + + //! Has space for levin header until a finalize method is used + byte_stream buffer; + }; + //! \return Intialized levin header. bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; - //! \return A levin notification message. - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); - /*! Generate a dummy levin message. \param noise_bytes Total size of the returned `byte_slice`. @@ -140,12 +171,11 @@ namespace levin /*! Generate 1+ levin messages that are identical to the noise message size. - \param noise Each levin message will be identical to the size of this - message. The bytes from this message will be used for padding. + \param noise_size Each levin message will be identical to this value. \return `nullptr` if `noise.size()` is less than the levin header size. Otherwise, a levin notification message OR 2+ levin fragment messages. Each message is `noise.size()` in length. */ - byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); + byte_slice make_fragmented_notify(const std::size_t noise_size, int command, message_writer message); } } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index ddde701ee..a6816cafc 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -51,6 +51,21 @@ #define MIN_BYTES_WANTED 512 #endif +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char* category) +{ + MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") + << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); +} + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "command-%u", command); + on_levin_traffic(context, initiator, sent, error, bytes, buf); +} + namespace epee { namespace levin @@ -88,11 +103,10 @@ public: uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id); template<class callback_t> - int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); @@ -122,12 +136,17 @@ class async_protocol_handler { std::string m_fragment_buffer; - bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + bool send_message(byte_slice message) { - const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); - if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + if (message.size() < sizeof(message_writer::header)) + return false; + + message_writer::header head; + std::memcpy(std::addressof(head), message.data(), sizeof(head)); + if(!m_pservice_endpoint->do_send(std::move(message))) return false; + on_levin_traffic(m_connection_context, true, true, false, head.m_cb, head.m_command); MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb << ", flags" << head.m_flags << ", r?=" << head.m_have_to_return_data @@ -146,7 +165,6 @@ public: stream_state_body }; - std::atomic<bool> m_deletion_initiated; std::atomic<bool> m_protocol_released; volatile uint32_t m_invoke_buf_ready; @@ -297,7 +315,6 @@ public: m_state(stream_state_head) { m_close_called = 0; - m_deletion_initiated = false; m_protocol_released = false; m_wait_count = 0; m_oponent_protocol_ver = 0; @@ -310,7 +327,6 @@ public: try { - m_deletion_initiated = true; if(m_connection_initialized) { m_config.del_connection(this); @@ -526,26 +542,17 @@ public: { if(m_current_head.m_have_to_return_data) { - byte_slice return_buff; + levin::message_writer return_message{32 * 1024}; const uint32_t return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + m_current_head.m_command, buff_to_invoke, return_message.buffer, m_connection_context ); // peer_id remains unset if dropped if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) m_max_packet_size = m_config.m_max_packet_size; - bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); - head.m_return_code = SWAP32LE(return_code); - - if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) + if(!send_message(return_message.finalize_response(m_current_head.m_command, return_code))) return false; - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", flags" << head.m_flags - << ", r?=" << head.m_have_to_return_data - <<", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); @@ -622,7 +629,7 @@ public: } template<class callback_t> - bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, message_writer in_msg, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -633,27 +640,15 @@ public: int err_code = LEVIN_OK; do { - if(m_deletion_initiated) - { - err_code = LEVIN_ERROR_CONNECTION_DESTROYED; - break; - } - CRITICAL_REGION_LOCAL(m_call_lock); - if(m_deletion_initiated) - { - err_code = LEVIN_ERROR_CONNECTION_DESTROYED; - break; - } - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock); if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if(!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -679,25 +674,19 @@ public: return true; } - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out) + int invoke(int command, message_writer in_msg, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - CRITICAL_REGION_LOCAL(m_call_lock); - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if (!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; @@ -706,7 +695,7 @@ public: uint64_t ticks_start = misc_utils::get_tick_count(); size_t prev_size = 0; - while(!boost::interprocess::ipcdetail::atomic_read32(&m_invoke_buf_ready) && !m_deletion_initiated && !m_protocol_released) + while(!boost::interprocess::ipcdetail::atomic_read32(&m_invoke_buf_ready) && !m_protocol_released) { if(m_cache_in_buffer.size() - prev_size >= MIN_BYTES_WANTED) { @@ -723,7 +712,7 @@ public: return LEVIN_ERROR_CONNECTION_DESTROYED; } - if(m_deletion_initiated || m_protocol_released) + if(m_protocol_released) return LEVIN_ERROR_CONNECTION_DESTROYED; CRITICAL_REGION_BEGIN(m_local_inv_buff_lock); @@ -734,31 +723,9 @@ public: return m_invoke_result_code; } - int notify(int command, const epee::span<const uint8_t> in_buff) - { - misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( - boost::bind(&async_protocol_handler::finish_outer_call, this)); - - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - - CRITICAL_REGION_LOCAL(m_call_lock); - - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) - { - LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); - return -1; - } - - return 1; - } - - /*! Sends `message` without adding a levin header. The message must have - been created with `make_notify`, `make_noise_notify` or - `make_fragmented_notify`. See additional instructions for + /*! Sends `message` without adding a levin header. The message must have been + created with `make_noise_notify`, `make_fragmented_notify`, or + `message_writer::finalize_notify`. See additional instructions for `make_fragmented_notify`. \return 1 on success */ @@ -768,17 +735,11 @@ public: boost::bind(&async_protocol_handler::finish_outer_call, this) ); - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - - const std::size_t length = message.size(); - if (!m_pservice_endpoint->do_send(std::move(message))) + if (!send_message(std::move(message))) { LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -799,36 +760,32 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::delete_connections(size_t count, bool incoming) { - std::vector <boost::uuids::uuid> connections; + std::vector<typename connections_map::mapped_type> connections; + + auto scope_exit_handler = misc_utils::create_scope_leave_handler([&connections]{ + for (auto &aph: connections) + aph->finish_outer_call(); + }); + CRITICAL_REGION_BEGIN(m_connects_lock); for (auto& c: m_connects) { if (c.second->m_connection_context.m_is_income == incoming) - connections.push_back(c.first); + if (c.second->start_outer_call()) + connections.push_back(c.second); } // close random connections from the provided set // TODO or better just keep removing random elements (performance) unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); shuffle(connections.begin(), connections.end(), std::default_random_engine(seed)); - while (count > 0 && connections.size() > 0) - { - try - { - auto i = connections.end() - 1; - async_protocol_handler<t_connection_context> *conn = m_connects.at(*i); - del_connection(conn); - conn->close(); - connections.erase(i); - } - catch (const std::out_of_range &e) - { - MWARNING("Connection not found in m_connects, continuing"); - } - --count; - } + for (size_t i = 0; i < connections.size() && i < count; ++i) + m_connects.erase(connections[i]->get_connection_id()); CRITICAL_REGION_END(); + + for (size_t i = 0; i < connections.size() && i < count; ++i) + connections[i]->close(); } //------------------------------------------------------------------------------------------ template<class t_connection_context> @@ -872,41 +829,53 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config<t_connection_context>::invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->invoke(command, in_buff, buff_out) : r; + return LEVIN_OK == r ? aph->invoke(command, std::move(in_msg), buff_out) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> -int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config<t_connection_context>::invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->async_invoke(command, in_buff, cb, timeout) : r; + return LEVIN_OK == r ? aph->async_invoke(command, std::move(in_msg), cb, timeout) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::foreach_connection(const callback_t &cb) { - CRITICAL_REGION_LOCAL(m_connects_lock); - for(auto& c: m_connects) - { - async_protocol_handler<t_connection_context>* aph = c.second; - if(!cb(aph->get_context_ref())) + std::vector<typename connections_map::mapped_type> conn; + + auto scope_exit_handler = misc_utils::create_scope_leave_handler([&conn]{ + for (auto &aph: conn) + aph->finish_outer_call(); + }); + + CRITICAL_REGION_BEGIN(m_connects_lock); + conn.reserve(m_connects.size()); + for (auto &e: m_connects) + if (e.second->start_outer_call()) + conn.push_back(e.second); + CRITICAL_REGION_END() + + for (auto &aph: conn) + if (!cb(aph->get_context_ref())) return false; - } + return true; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::for_connection(const boost::uuids::uuid &connection_id, const callback_t &cb) { - CRITICAL_REGION_LOCAL(m_connects_lock); - async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); - if (!aph) + async_protocol_handler<t_connection_context>* aph = nullptr; + if (find_and_lock_connection(connection_id, aph) != LEVIN_OK) return false; + auto scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); if(!cb(aph->get_context_ref())) return false; return true; @@ -951,14 +920,6 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id) -{ - async_protocol_handler<t_connection_context>* aph; - int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->notify(command, in_buff) : r; -} -//------------------------------------------------------------------------------------------ -template<class t_connection_context> int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) { async_protocol_handler<t_connection_context>* aph; @@ -969,12 +930,14 @@ int async_protocol_handler_config<t_connection_context>::send(byte_slice message template<class t_connection_context> bool async_protocol_handler_config<t_connection_context>::close(boost::uuids::uuid connection_id) { - CRITICAL_REGION_LOCAL(m_connects_lock); - async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); - if (!aph) + async_protocol_handler<t_connection_context>* aph = nullptr; + if (find_and_lock_connection(connection_id, aph) != LEVIN_OK) return false; + auto scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); if (!aph->close()) return false; + CRITICAL_REGION_LOCAL(m_connects_lock); m_connects.erase(connection_id); return true; } diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 1b1577e77..58cd7e45f 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -36,6 +36,7 @@ #include <boost/utility/string_ref.hpp> #include <boost/asio/ip/tcp.hpp> #include <boost/asio/ssl.hpp> +#include <boost/filesystem/path.hpp> #include <boost/system/error_code.hpp> #define SSL_FINGERPRINT_SIZE 32 @@ -144,6 +145,9 @@ namespace net_utils bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + + //! Store private key for `ssl` at `base + ".key"` unencrypted and certificate for `ssl` at `base + ".crt"`. + boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base); } } diff --git a/contrib/epee/include/serialization/keyvalue_serialization.h b/contrib/epee/include/serialization/keyvalue_serialization.h index fd343865c..2e4a0faad 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization.h +++ b/contrib/epee/include/serialization/keyvalue_serialization.h @@ -73,7 +73,8 @@ public: \ template<bool is_store, class t_storage> \ bool serialize_map(t_storage& stg, typename t_storage::hsection hparent_section) \ { \ - decltype(*this) &this_ref = *this; + decltype(*this) &this_ref = *this; \ + (void) this_ref; // Suppress unused var warnings. Sometimes this var is used, sometimes not. #define KV_SERIALIZE_N(varialble, val_name) \ epee::serialization::selector<is_store>::serialize(this_ref.varialble, stg, hparent_section, val_name); diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 802e16c1b..383d67cc2 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -37,21 +37,14 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category); + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command); + namespace { - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category) - { - MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") - << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); - } - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) - { - char buf[32]; - snprintf(buf, sizeof(buf), "command-%u", command); - return on_levin_traffic(context, initiator, sent, error, bytes, buf); - } static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { 8192, // objects 16384, // fields @@ -117,12 +110,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; + levin::message_writer to_send{16 * 1024}; std::string buff_to_recv; - stg.store_to_binary(buff_to_send, 16 * 1024); + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); + int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); @@ -145,10 +137,9 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send, 16 * 1024); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + levin::message_writer to_send{16 * 1024}; + stg.store_to_binary(to_send.buffer); + int res = transport.invoke_async(command, std::move(to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -192,11 +183,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send); + levin::message_writer to_send; + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); + int res = transport.send(to_send.finalize_notify(command), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -207,7 +197,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff, &default_levin_limits)) @@ -230,12 +220,11 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out, 32 * 1024)) + if(!strg_out.store_to_binary(buff_out)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; } - on_levin_traffic(context, false, true, false, buff_out.size(), command); return res; } @@ -262,7 +251,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -271,13 +260,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; epee::byte_slice fake_str; \ - return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ + bool handled = false; epee::byte_stream fake_str; \ + return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -297,7 +286,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index f77e89cb6..655a2eb12 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -24,24 +24,17 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #pragma once -#include <type_traits> - -#include "misc_language.h" #include "portable_storage_base.h" -#include "portable_storage_from_bin.h" -#include "portable_storage_to_json.h" -#include "portable_storage_from_json.h" #include "portable_storage_val_converters.h" +#include "misc_log_ex.h" #include "span.h" -#include "int-util.h" namespace epee { class byte_slice; + class byte_stream; namespace serialization { /************************************************************************/ @@ -91,8 +84,13 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); - bool load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); } + bool store_to_binary(byte_stream& ss); + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = nullptr); + bool load_from_binary(const std::string& target, const limits_t *limits = nullptr) + { + return load_from_binary(epee::strspan<uint8_t>(target), limits); + } + template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -117,85 +115,13 @@ namespace epee }; #pragma pack(pop) }; - inline - bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) - { - TRY_ENTRY(); - std::stringstream ss; - epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines); - buff = ss.str(); - return true; - CATCH_ENTRY("portable_storage::dump_as_json", false) - } - inline - bool portable_storage::load_from_json(const std::string& source) - { - TRY_ENTRY(); - return json::load_from_json(source, *this); - CATCH_ENTRY("portable_storage::load_from_json", false) - } - + template<class trace_policy> bool portable_storage::dump_as_xml(std::string& targetObj, const std::string& root_name) { return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format - } - inline - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) - { - m_root.m_entries.clear(); - if(source.size() < sizeof(storage_block_header)) - { - LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); - return false; - } - storage_block_header* pbuff = (storage_block_header*)source.data(); - if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || - pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) - ) - { - LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); - return false; - } - if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) - { - LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); - return false; - } - TRY_ENTRY(); - throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); - if (limits) - buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); - buf_reader.read(m_root); - return true;//TODO: - CATCH_ENTRY("portable_storage::load_from_binary", false); - } - //--------------------------------------------------------------------------------------------------------------- - inline - hsection portable_storage::open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist) - { - TRY_ENTRY(); - hparent_section = hparent_section ? hparent_section:&m_root; - storage_entry* pentry = find_storage_entry(section_name, hparent_section); - if(!pentry) - { - if(!create_if_notexist) - return nullptr; - return insert_new_section(section_name, hparent_section); - } - CHECK_AND_ASSERT(pentry , nullptr); - //check that section_entry we find is real "CSSection" - if(pentry->type() != typeid(section)) - { - if(create_if_notexist) - *pentry = storage_entry(section());//replace - else - return nullptr; - } - return &boost::get<section>(*pentry); - CATCH_ENTRY("portable_storage::open_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- + } + template<class to_type> struct get_value_visitor: boost::static_visitor<void> { @@ -221,20 +147,6 @@ namespace epee //CATCH_ENTRY("portable_storage::template<>get_value", false); } //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section) - { - //TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(value_name, hparent_section); - if(!pentry) - return false; - - val = *pentry; - return true; - //CATCH_ENTRY("portable_storage::template<>get_value", false); - } - //--------------------------------------------------------------------------------------------------------------- template<class t_value> bool portable_storage::set_value(const std::string& value_name, t_value&& v, hsection hparent_section) { @@ -256,19 +168,6 @@ namespace epee CATCH_ENTRY("portable_storage::template<>set_value", false); } //--------------------------------------------------------------------------------------------------------------- - inline - storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(psection, nullptr); - auto it = psection->m_entries.find(pentry_name); - if(it == psection->m_entries.end()) - return nullptr; - - return &it->second; - CATCH_ENTRY("portable_storage::find_storage_entry", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- template<class entry_type> storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry) { @@ -281,16 +180,6 @@ namespace epee CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr); } //--------------------------------------------------------------------------------------------------------------- - inline - hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection) - { - TRY_ENTRY(); - storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section()); - if(!pse) return nullptr; - return &boost::get<section>(*pse); - CATCH_ENTRY("portable_storage::insert_new_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- template<class to_type> struct get_first_value_visitor: boost::static_visitor<bool> { @@ -344,7 +233,6 @@ namespace epee } }; - template<class t_value> bool portable_storage::get_next_value(harray hval_array, t_value& target) { @@ -402,83 +290,5 @@ namespace epee return true; CATCH_ENTRY("portable_storage::insert_next_value", false); } - //--------------------------------------------------------------------------------------------------------------- - //sections - inline - harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section) - { - TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(sec_name, hparent_section); - if(!pentry) - return nullptr; - if(pentry->type() != typeid(array_entry)) - return nullptr; - array_entry& ar_entry = boost::get<array_entry>(*pentry); - if(ar_entry.type() != typeid(array_entry_t<section>)) - return nullptr; - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); - section* psec = sec_array.get_first_val(); - if(!psec) - return nullptr; - h_child_section = psec; - return &ar_entry; - CATCH_ENTRY("portable_storage::get_first_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(hsec_array, false); - if(hsec_array->type() != typeid(array_entry_t<section>)) - return false; - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); - h_child_section = sec_array.get_next_val(); - if(!h_child_section) - return false; - return true; - CATCH_ENTRY("portable_storage::get_next_section", false); - } - //--------------------------------------------------------------------------------------------------------------- - inline - harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section) - { - TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(sec_name, hparent_section); - if(!pentry) - { - pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>())); - if(!pentry) - return nullptr; - } - if(pentry->type() != typeid(array_entry)) - *pentry = storage_entry(array_entry(array_entry_t<section>())); - - array_entry& ar_entry = boost::get<array_entry>(*pentry); - if(ar_entry.type() != typeid(array_entry_t<section>)) - ar_entry = array_entry(array_entry_t<section>()); - - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); - hinserted_childsection = &sec_array.insert_first_val(section()); - return &ar_entry; - CATCH_ENTRY("portable_storage::insert_first_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(hsec_array, false); - CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), - false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name()); - - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); - hinserted_childsection = &sec_array.insert_next_value(section()); - return true; - CATCH_ENTRY("portable_storage::insert_next_section", false); - } - //--------------------------------------------------------------------------------------------------------------- } } diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index 1676f41fb..ae0be6a34 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -29,10 +29,10 @@ #pragma once #include <boost/variant.hpp> -#include <boost/any.hpp> #include <string> #include <vector> #include <deque> +#include <map> #define PORTABLE_STORAGE_SIGNATUREA 0x01011101 #define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 2b2dc7ff9..3021598f5 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -26,6 +26,7 @@ #pragma once #include <boost/lexical_cast.hpp> +#include <boost/utility/string_ref.hpp> #include <boost/algorithm/string/predicate.hpp> #include "parserse_base_utils.h" #include "file_io_utils.h" diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 16dd565ec..7f6596f36 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -29,12 +29,15 @@ #include <string> #include "byte_slice.h" -#include "parserse_base_utils.h" +#include "parserse_base_utils.h" /// TODO: (mj-xmr) This will be reduced in an another PR #include "portable_storage.h" #include "file_io_utils.h" +#include "span.h" namespace epee { + class byte_stream; + namespace serialization { //----------------------------------------------------------------------------------------------------------- @@ -126,5 +129,14 @@ namespace epee store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } + //----------------------------------------------------------------------------------------------------------- + template<class t_struct> + bool store_t_to_binary(t_struct& str_in, byte_stream& binary_buff) + { + portable_storage ps; + str_in.store(ps); + return ps.store_to_binary(binary_buff); + } + } } diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 49a7be185..b82cf532b 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -32,6 +32,7 @@ #include "misc_language.h" #include "portable_storage_base.h" #include "portable_storage_bin_utils.h" +#include "misc_log_ex.h" namespace epee { diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index e54cda828..96b0c024c 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -28,12 +28,17 @@ #pragma once -#include <time.h> #include <boost/regex.hpp> #include "misc_language.h" #include "portable_storage_base.h" +#include "parserse_base_utils.h" #include "warnings.h" +#include "misc_log_ex.h" + +#include <boost/lexical_cast.hpp> +#include <typeinfo> +#include <iomanip> namespace epee { diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 5e101a86a..132fed355 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,10 +26,21 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set(EPEE_INCLUDE_DIR_BASE "${CMAKE_CURRENT_SOURCE_DIR}/../include") + +# Adding headers to the file list, to be able to search for them in IDEs. +file(GLOB EPEE_HEADERS_PUBLIC + "${EPEE_INCLUDE_DIR_BASE}/*.h*" # h* will include hpps as well. + "${EPEE_INCLUDE_DIR_BASE}/**/*.h*" # Any number of subdirs will be included. +) add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp - int-util.cpp portable_storage.cpp) + int-util.cpp portable_storage.cpp + misc_language.cpp + file_io_utils.cpp + ${EPEE_HEADERS_PUBLIC} + ) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) add_library(epee_readline STATIC readline_buffer.cpp) @@ -71,3 +82,6 @@ if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) PRIVATE ${GNU_READLINE_LIBRARY}) endif() + +target_include_directories(epee PUBLIC "${EPEE_INCLUDE_DIR_BASE}") + diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp index faf7689be..453b63a4c 100644 --- a/contrib/epee/src/byte_slice.cpp +++ b/contrib/epee/src/byte_slice.cpp @@ -36,6 +36,11 @@ #include "byte_slice.h" #include "byte_stream.h" +namespace +{ + const std::size_t page_size = 4096; +} + namespace epee { struct byte_slice_data @@ -173,16 +178,27 @@ namespace epee : byte_slice(adapt_buffer{}, std::move(buffer)) {} - byte_slice::byte_slice(byte_stream&& stream) noexcept + byte_slice::byte_slice(byte_stream&& stream, const bool shrink) : storage_(nullptr), portion_(stream.data(), stream.size()) { - if (stream.size()) + if (portion_.size()) { - std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice); + byte_buffer buf; + if (shrink && page_size <= stream.available()) + { + buf = byte_buffer_resize(stream.take_buffer(), portion_.size()); + if (!buf) + throw std::bad_alloc{}; + portion_ = {buf.get(), portion_.size()}; + } + else // no need to shrink buffer + buf = stream.take_buffer(); + + std::uint8_t* const data = buf.release() - sizeof(raw_byte_slice); new (data) raw_byte_slice{}; storage_.reset(reinterpret_cast<raw_byte_slice*>(data)); } - else + else // empty stream portion_ = nullptr; } diff --git a/contrib/epee/src/file_io_utils.cpp b/contrib/epee/src/file_io_utils.cpp new file mode 100644 index 000000000..5072adcbc --- /dev/null +++ b/contrib/epee/src/file_io_utils.cpp @@ -0,0 +1,231 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "file_io_utils.h" + +#include <fstream> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> +#ifdef WIN32 +#include <windows.h> +#include "string_tools.h" +#endif + +// On Windows there is a problem with non-ASCII characters in path and file names +// as far as support by the standard components used is concerned: + +// The various file stream classes, e.g. std::ifstream and std::ofstream, are +// part of the GNU C++ Library / libstdc++. On the most basic level they use the +// fopen() call as defined / made accessible to programs compiled within MSYS2 +// by the stdio.h header file maintained by the MinGW project. + +// The critical point: The implementation of fopen() is part of MSVCRT, the +// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any +// Unicode support. + +// Monero code that would want to continue to use the normal file stream classes +// but WITH Unicode support could therefore not solve this problem on its own, +// but 2 different projects from 2 different maintaining groups would need changes +// in this particular direction - something probably difficult to achieve and +// with a long time to wait until all new versions / releases arrive. + +// Implemented solution approach: Circumvent the problem by stopping to use std +// file stream classes on Windows and directly use Unicode-capable WIN32 API +// calls. Most of the code doing so is concentrated in this header file here. + +namespace epee +{ +namespace file_io_utils +{ + + bool is_file_exist(const std::string& path) + { + boost::filesystem::path p(path); + return boost::filesystem::exists(p); + } + + + bool save_string_to_file(const std::string& path_to_file, const std::string& str) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD bytes_written; + DWORD bytes_to_write = (DWORD)str.size(); + BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); + CloseHandle(file_handle); + if (bytes_written != bytes_to_write) + result = FALSE; + return result; +#else + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool get_file_time(const std::string& path_to_file, time_t& ft) + { + boost::system::error_code ec; + ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); + if(!ec) + return true; + else + return false; + } + + + bool set_file_time(const std::string& path_to_file, const time_t& ft) + { + boost::system::error_code ec; + boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); + if(!ec) + return true; + else + return false; + } + + + + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD file_size = GetFileSize(file_handle, NULL); + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { + CloseHandle(file_handle); + return false; + } + target_str.resize(file_size); + DWORD bytes_read; + BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); + CloseHandle(file_handle); + if (bytes_read != file_size) + result = FALSE; + return result; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + + std::ifstream::pos_type file_size = fstream.tellg(); + + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large + return false;//don't go crazy + size_t file_size_t = static_cast<size_t>(file_size); + + target_str.resize(file_size_t); + + fstream.seekg (0, std::ios::beg); + fstream.read((char*)target_str.data(), target_str.size()); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool append_string_to_file(const std::string& path_to_file, const std::string& str) + { + // No special Windows implementation because so far not used in Monero code + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } + } + + + bool get_file_size(const std::string& path_to_file, uint64_t &size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER file_size; + BOOL result = GetFileSizeEx(file_handle, &file_size); + CloseHandle(file_handle); + if (result) { + size = file_size.QuadPart; + } + return size; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + size = fstream.tellg(); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + +} +} diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp index 5ec86b3d6..7c5cd5a78 100644 --- a/contrib/epee/src/levin_base.cpp +++ b/contrib/epee/src/levin_base.cpp @@ -34,6 +34,25 @@ namespace epee { namespace levin { + message_writer::message_writer(const std::size_t reserve) + : buffer() + { + buffer.reserve(reserve); + buffer.put_n(0, sizeof(header)); + } + + byte_slice message_writer::finalize(const uint32_t command, const uint32_t flags, const uint32_t return_code, const bool expect_response) + { + if (buffer.size() < sizeof(header)) + throw std::runtime_error{"levin_writer::finalize already called"}; + + header head = make_header(command, payload_size(), flags, expect_response); + head.m_return_code = SWAP32LE(return_code); + + std::memcpy(buffer.tellp() - buffer.size(), std::addressof(head), sizeof(head)); + return byte_slice{std::move(buffer)}; + } + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept { bucket_head2 head = {0}; @@ -47,12 +66,6 @@ namespace levin return head; } - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) - { - const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); - return byte_slice{epee::as_byte_span(head), payload}; - } - byte_slice make_noise_notify(const std::size_t noise_bytes) { static constexpr const std::uint32_t flags = @@ -68,46 +81,40 @@ namespace levin return byte_slice{std::move(buffer)}; } - byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + byte_slice make_fragmented_notify(const std::size_t noise_size, const int command, message_writer message) { - const size_t noise_size = noise_message.size(); if (noise_size < sizeof(bucket_head2) * 2) return nullptr; - if (payload.size() <= noise_size - sizeof(bucket_head2)) + if (message.buffer.size() <= noise_size) { /* The entire message can be sent at once, and the levin binary parser will ignore extra bytes. So just pad with zeroes and otherwise send a "normal", not fragmented message. */ - const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); - const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; - const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); - return byte_slice{as_byte_span(head), payload, padding_bytes}; + message.buffer.put_n(0, noise_size - message.buffer.size()); + return message.finalize_notify(command); } // fragment message + const byte_slice payload_bytes = message.finalize_notify(command); + span<const std::uint8_t> payload = to_span(payload_bytes); + const size_t payload_space = noise_size - sizeof(bucket_head2); const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; - std::string buffer{}; - buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + byte_stream buffer{}; + buffer.reserve(expected_fragments * noise_size); - bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + bucket_head2 head = make_header(0, payload_space, LEVIN_PACKET_BEGIN, false); + buffer.write(as_byte_span(head)); - head.m_command = command; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_cb = payload.size(); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + // internal levin header is in payload already - size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + size_t copy_size = payload.remove_prefix(payload_space); + buffer.write(payload.data() - copy_size, copy_size); - head.m_command = 0; head.m_flags = 0; - head.m_cb = noise_size - sizeof(bucket_head2); - while (!payload.empty()) { copy_size = payload.remove_prefix(payload_space); @@ -115,12 +122,12 @@ namespace levin if (payload.empty()) head.m_flags = LEVIN_PACKET_END; - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + buffer.write(as_byte_span(head)); + buffer.write(payload.data() - copy_size, copy_size); } const size_t padding = noise_size - copy_size - sizeof(bucket_head2); - buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + buffer.put_n(0, padding); return byte_slice{std::move(buffer)}; } diff --git a/contrib/epee/src/misc_language.cpp b/contrib/epee/src/misc_language.cpp new file mode 100644 index 000000000..6e8f2daef --- /dev/null +++ b/contrib/epee/src/misc_language.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_language.h" + +#include <boost/thread.hpp> + +namespace epee +{ +namespace misc_utils +{ + bool sleep_no_w(long ms ) + { + boost::this_thread::sleep( + boost::get_system_time() + + boost::posix_time::milliseconds( std::max<long>(ms,0) ) ); + + return true; + } +} +} diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index 6ed27efa9..765dadce3 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -29,6 +29,8 @@ #include <string.h> #include <thread> #include <boost/asio/ssl.hpp> +#include <boost/cerrno.hpp> +#include <boost/filesystem/operations.hpp> #include <boost/lambda/lambda.hpp> #include <openssl/ssl.h> #include <openssl/pem.h> @@ -567,6 +569,51 @@ bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s) return true; } +boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base) +{ + EVP_PKEY* ssl_key = nullptr; + X509* ssl_cert = nullptr; + const auto ctx = ssl.native_handle(); + CHECK_AND_ASSERT_MES(ctx, boost::system::error_code(EINVAL, boost::system::system_category()), "Context is null"); + CHECK_AND_ASSERT_MES(base.has_filename(), boost::system::error_code(EINVAL, boost::system::system_category()), "Need filename"); + if (!(ssl_key = SSL_CTX_get0_privatekey(ctx)) || !(ssl_cert = SSL_CTX_get0_certificate(ctx))) + return {EINVAL, boost::system::system_category()}; + + using file_closer = int(std::FILE*); + boost::system::error_code error{}; + std::unique_ptr<std::FILE, file_closer*> file{nullptr, std::fclose}; + + // write key file unencrypted + { + const boost::filesystem::path key_file{base.string() + ".key"}; + file.reset(std::fopen(key_file.string().c_str(), "wb")); + if (!file) + return {errno, boost::system::system_category()}; + boost::filesystem::permissions(key_file, boost::filesystem::owner_read, error); + if (error) + return error; + if (!PEM_write_PrivateKey(file.get(), ssl_key, nullptr, nullptr, 0, nullptr, nullptr)) + return boost::asio::error::ssl_errors(ERR_get_error()); + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + } + + // write certificate file in standard SSL X.509 unencrypted + const boost::filesystem::path cert_file{base.string() + ".crt"}; + file.reset(std::fopen(cert_file.string().c_str(), "wb")); + if (!file) + return {errno, boost::system::system_category()}; + const auto cert_perms = (boost::filesystem::owner_read | boost::filesystem::group_read | boost::filesystem::others_read); + boost::filesystem::permissions(cert_file, cert_perms, error); + if (error) + return error; + if (!PEM_write_X509(file.get(), ssl_cert)) + return boost::asio::error::ssl_errors(ERR_get_error()); + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + return error; +} + } // namespace } // namespace diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index 4534deff3..b922cc9e3 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -1,10 +1,43 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #include "byte_slice.h" #include "byte_stream.h" #include "misc_log_ex.h" #include "span.h" #include "storages/portable_storage.h" +#include "storages/portable_storage_to_json.h" +#include "storages/portable_storage_from_json.h" #include "storages/portable_storage_to_bin.h" +#include "storages/portable_storage_from_bin.h" + +#include <boost/utility/string_ref.hpp> + +#include <string> +#include <sstream> namespace epee { @@ -15,15 +48,200 @@ namespace serialization TRY_ENTRY(); byte_stream ss; ss.reserve(initial_buffer_size); + store_to_binary(ss); + target = epee::byte_slice{std::move(ss)}; + return true; + CATCH_ENTRY("portable_storage::store_to_binary", false); + } + + bool portable_storage::store_to_binary(byte_stream& ss) + { + TRY_ENTRY(); storage_block_header sbh{}; sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; ss.write(epee::as_byte_span(sbh)); pack_entry_to_buff(ss, m_root); - target = epee::byte_slice{std::move(ss)}; return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) + CATCH_ENTRY("portable_storage::store_to_binary", false); } + + bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) + { + TRY_ENTRY(); + std::stringstream ss; + epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines); + buff = ss.str(); + return true; + CATCH_ENTRY("portable_storage::dump_as_json", false) + } + + bool portable_storage::load_from_json(const std::string& source) + { + TRY_ENTRY(); + return json::load_from_json(source, *this); + CATCH_ENTRY("portable_storage::load_from_json", false) + } + + bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) + { + m_root.m_entries.clear(); + if(source.size() < sizeof(storage_block_header)) + { + LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); + return false; + } + storage_block_header* pbuff = (storage_block_header*)source.data(); + if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || + pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) + ) + { + LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); + return false; + } + if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) + { + LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); + return false; + } + TRY_ENTRY(); + throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); + if (limits) + buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); + buf_reader.read(m_root); + return true;//TODO: + CATCH_ENTRY("portable_storage::load_from_binary", false); + } + + hsection portable_storage::open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist) + { + TRY_ENTRY(); + hparent_section = hparent_section ? hparent_section:&m_root; + storage_entry* pentry = find_storage_entry(section_name, hparent_section); + if(!pentry) + { + if(!create_if_notexist) + return nullptr; + return insert_new_section(section_name, hparent_section); + } + CHECK_AND_ASSERT(pentry , nullptr); + //check that section_entry we find is real "CSSection" + if(pentry->type() != typeid(section)) + { + if(create_if_notexist) + *pentry = storage_entry(section());//replace + else + return nullptr; + } + return &boost::get<section>(*pentry); + CATCH_ENTRY("portable_storage::open_section", nullptr); + } + + bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section) + { + //TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(value_name, hparent_section); + if(!pentry) + return false; + + val = *pentry; + return true; + //CATCH_ENTRY("portable_storage::template<>get_value", false); + } + + storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(psection, nullptr); + auto it = psection->m_entries.find(pentry_name); + if(it == psection->m_entries.end()) + return nullptr; + + return &it->second; + CATCH_ENTRY("portable_storage::find_storage_entry", nullptr); + } + + hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection) + { + TRY_ENTRY(); + storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section()); + if(!pse) return nullptr; + return &boost::get<section>(*pse); + CATCH_ENTRY("portable_storage::insert_new_section", nullptr); + } + + harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section) + { + TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(sec_name, hparent_section); + if(!pentry) + return nullptr; + if(pentry->type() != typeid(array_entry)) + return nullptr; + array_entry& ar_entry = boost::get<array_entry>(*pentry); + if(ar_entry.type() != typeid(array_entry_t<section>)) + return nullptr; + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); + section* psec = sec_array.get_first_val(); + if(!psec) + return nullptr; + h_child_section = psec; + return &ar_entry; + CATCH_ENTRY("portable_storage::get_first_section", nullptr); + } + + bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(hsec_array, false); + if(hsec_array->type() != typeid(array_entry_t<section>)) + return false; + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); + h_child_section = sec_array.get_next_val(); + if(!h_child_section) + return false; + return true; + CATCH_ENTRY("portable_storage::get_next_section", false); + } + + harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section) + { + TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(sec_name, hparent_section); + if(!pentry) + { + pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>())); + if(!pentry) + return nullptr; + } + if(pentry->type() != typeid(array_entry)) + *pentry = storage_entry(array_entry(array_entry_t<section>())); + + array_entry& ar_entry = boost::get<array_entry>(*pentry); + if(ar_entry.type() != typeid(array_entry_t<section>)) + ar_entry = array_entry(array_entry_t<section>()); + + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); + hinserted_childsection = &sec_array.insert_first_val(section()); + return &ar_entry; + CATCH_ENTRY("portable_storage::insert_first_section", nullptr); + } + + bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(hsec_array, false); + CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), + false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name()); + + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); + hinserted_childsection = &sec_array.insert_next_value(section()); + return true; + CATCH_ENTRY("portable_storage::insert_next_section", false); + } } } |