diff options
Diffstat (limited to 'contrib/epee/include')
18 files changed, 204 insertions, 105 deletions
diff --git a/contrib/epee/include/byte_stream.h b/contrib/epee/include/byte_stream.h index 42a9e1dd9..93f9ac85c 100644 --- a/contrib/epee/include/byte_stream.h +++ b/contrib/epee/include/byte_stream.h @@ -58,7 +58,6 @@ namespace epee byte_buffer buffer_; //! Beginning of buffer std::uint8_t* next_write_; //! Current write position const std::uint8_t* end_; //! End of buffer - std::size_t increase_size_; //! Minimum buffer size increase //! \post `requested <= available()` void overflow(const std::size_t requested); @@ -75,29 +74,17 @@ namespace epee using char_type = std::uint8_t; using Ch = char_type; - //! \return Default minimum size increase on buffer overflow - static constexpr std::size_t default_increase() noexcept { return 4096; } - //! Increase internal buffer by at least `byte_stream_increase` bytes. byte_stream() noexcept - : byte_stream(default_increase()) - {} - - //! Increase internal buffer by at least `increase` bytes. - explicit byte_stream(const std::size_t increase) noexcept : buffer_(nullptr), next_write_(nullptr), - end_(nullptr), - increase_size_(increase) + end_(nullptr) {} byte_stream(byte_stream&& rhs) noexcept; ~byte_stream() noexcept = default; byte_stream& operator=(byte_stream&& rhs) noexcept; - //! \return The minimum increase size on buffer overflow - std::size_t increase_size() const noexcept { return increase_size_; } - const std::uint8_t* data() const noexcept { return buffer_.get(); } std::uint8_t* tellp() const noexcept { return next_write_; } std::size_t available() const noexcept { return end_ - next_write_; } @@ -188,7 +175,7 @@ namespace epee void put_n(const std::uint8_t ch, const std::size_t count) { check(count); - std::memset(tellp(), count, ch); + std::memset(tellp(), ch, count); next_write_ += count; } diff --git a/contrib/epee/include/net/abstract_http_client.h b/contrib/epee/include/net/abstract_http_client.h index 1f8bbc605..5270824e1 100644 --- a/contrib/epee/include/net/abstract_http_client.h +++ b/contrib/epee/include/net/abstract_http_client.h @@ -70,7 +70,7 @@ namespace http virtual bool connect(std::chrono::milliseconds timeout) = 0; virtual bool disconnect() = 0; virtual bool is_connected(bool *ssl = NULL) = 0; - virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; + virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; virtual uint64_t get_bytes_sent() const = 0; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index cbacd118c..cb1388f3b 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -207,7 +207,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) buffer_ssl_init_fill = 0; if (is_income && m_ssl_support != epee::net_utils::ssl_support_t::e_ssl_support_disabled) socket().async_receive(boost::asio::buffer(buffer_), - boost::asio::socket_base::message_peek, strand_.wrap( std::bind(&connection<t_protocol_handler>::handle_receive, self, std::placeholders::_1, @@ -334,6 +333,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) TRY_ENTRY(); //_info("[sock " << socket().native_handle() << "] Async read calledback."); + if (m_was_shutdown) + return; + if (!e) { double current_speed_down; @@ -360,6 +362,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); } + + if (m_was_shutdown) + return; delay *= 0.5; long int ms = (long int)(delay * 100); @@ -431,6 +436,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) std::size_t bytes_transferred) { TRY_ENTRY(); + + if (m_was_shutdown) return; + if (e) { // offload the error case @@ -438,13 +446,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) return; } - reset_timer(get_timeout_from_bytes_read(bytes_transferred), false); - buffer_ssl_init_fill += bytes_transferred; - if (buffer_ssl_init_fill <= get_ssl_magic_size()) + MTRACE("we now have " << buffer_ssl_init_fill << "/" << get_ssl_magic_size() << " bytes needed to detect SSL"); + if (buffer_ssl_init_fill < get_ssl_magic_size()) { socket().async_receive(boost::asio::buffer(buffer_.data() + buffer_ssl_init_fill, buffer_.size() - buffer_ssl_init_fill), - boost::asio::socket_base::message_peek, strand_.wrap( boost::bind(&connection<t_protocol_handler>::handle_receive, connection<t_protocol_handler>::shared_from_this(), boost::asio::placeholders::error, @@ -470,7 +476,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled) { // Handshake - if (!handshake(boost::asio::ssl::stream_base::server)) + if (!handshake(boost::asio::ssl::stream_base::server, boost::asio::const_buffer(buffer_.data(), buffer_ssl_init_fill))) { MERROR("SSL handshake failed"); boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); @@ -485,6 +491,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) return; } } + else + { + handle_read(e, buffer_ssl_init_fill); + return; + } async_read_some(boost::asio::buffer(buffer_), strand_.wrap( @@ -651,6 +662,8 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); m_send_que_lock.lock(); _dbg1("sleep for queue: " << ms); + if (m_was_shutdown) + return false; if (retry > retry_limit) { MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); @@ -748,7 +761,8 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add) { - if (ms.total_milliseconds() < 0) + const auto tms = ms.total_milliseconds(); + if (tms < 0 || (add && tms == 0)) { MWARNING("Ignoring negative timeout " << ms); return; diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp index 90303a785..23873f65b 100644 --- a/contrib/epee/include/net/connection_basic.hpp +++ b/contrib/epee/include/net/connection_basic.hpp @@ -132,10 +132,10 @@ class connection_basic { // not-templated base class for rapid developmet of som ssl_support_t get_ssl_support() const { return m_ssl_support; } void disable_ssl() { m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; } - bool handshake(boost::asio::ssl::stream_base::handshake_type type) + bool handshake(boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer = {}) { //m_state != nullptr verified in constructor - return m_state->ssl_options().handshake(socket_, type); + return m_state->ssl_options().handshake(socket_, type, buffer); } template<typename MutableBufferSequence, typename ReadHandler> diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 9645e896b..3725ef079 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -233,7 +233,7 @@ namespace net_utils } //--------------------------------------------------------------------------- - inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override + inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override { CRITICAL_REGION_LOCAL(m_lock); if(!is_connected()) diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index ac22cd7a9..1665fdac7 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -118,8 +118,10 @@ return true; \ } \ uint64_t ticks2 = misc_utils::get_tick_count(); \ - epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), response_info.m_body); \ + epee::byte_slice buffer; \ + epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), buffer, 64 * 1024); \ uint64_t ticks3 = epee::misc_utils::get_tick_count(); \ + response_info.m_body.assign(reinterpret_cast<const char*>(buffer.data()), buffer.size()); \ response_info.m_mime_tipe = " application/octet-stream"; \ response_info.m_header_info.m_content_type = " application/octet-stream"; \ MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \ diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index f9b6f9a81..fce6d4b7e 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,7 +31,6 @@ #include <cstdint> -#include "byte_slice.h" #include "net_utils_base.h" #include "span.h" @@ -39,6 +38,7 @@ namespace epee { +class byte_slice; namespace levin { #pragma pack(push) @@ -72,7 +72,8 @@ namespace levin #define LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED 0 -#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000 //100MB by default +#define LEVIN_INITIAL_MAX_PACKET_SIZE 256*1024 // 256 KiB before handshake +#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000 //100MB by default after handshake #define LEVIN_PACKET_REQUEST 0x00000001 #define LEVIN_PACKET_RESPONSE 0x00000002 @@ -86,7 +87,7 @@ namespace levin template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 1341a4ae6..ddde701ee 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -84,7 +84,8 @@ class async_protocol_handler_config public: typedef t_connection_context connection_context; - uint64_t m_max_packet_size; + uint64_t m_initial_max_packet_size; + uint64_t m_max_packet_size; uint64_t m_invoke_timeout; int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); @@ -105,7 +106,7 @@ public: size_t get_in_connections_count(); void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL); - async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_initial_max_packet_size(LEVIN_INITIAL_MAX_PACKET_SIZE), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) {} ~async_protocol_handler_config() { set_handler(NULL, NULL); } void del_out_connections(size_t count); @@ -162,6 +163,7 @@ public: net_utils::i_service_endpoint* m_pservice_endpoint; config_type& m_config; t_connection_context& m_connection_context; + std::atomic<uint64_t> m_max_packet_size; net_utils::buffer m_cache_in_buffer; stream_state m_state; @@ -289,7 +291,8 @@ public: m_current_head(bucket_head2()), m_pservice_endpoint(psnd_hndlr), m_config(config), - m_connection_context(conn_context), + m_connection_context(conn_context), + m_max_packet_size(config.m_initial_max_packet_size), m_cache_in_buffer(4 * 1024), m_state(stream_state_head) { @@ -399,13 +402,14 @@ public: } // these should never fail, but do runtime check for safety - CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()"); - CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()"); + const uint64_t max_packet_size = m_max_packet_size; + CHECK_AND_ASSERT_MES(max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()"); + CHECK_AND_ASSERT_MES(max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()"); // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public - if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size()) + if(cb > max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size()) { - MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size + MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << max_packet_size << ", packet received " << m_cache_in_buffer.size() + cb << ", connection will be closed."); return false; @@ -430,7 +434,7 @@ public: //async call scenario boost::shared_ptr<invoke_response_handler_base> response_handler = m_invoke_response_handlers.front(); response_handler->reset_timer(); - MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb); + MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb << ", current total " << m_cache_in_buffer.size() << "/" << m_current_head.m_cb << " (" << (100.0f * m_cache_in_buffer.size() / (m_current_head.m_cb ? m_current_head.m_cb : 1)) << "%)"); } } break; @@ -465,6 +469,14 @@ public: temp = std::move(m_fragment_buffer); m_fragment_buffer.clear(); std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2)); + const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command); + if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes)) + { + MERROR(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes) + << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command + << ", connection will be closed."); + return false; + } buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)}; } @@ -514,16 +526,19 @@ public: { if(m_current_head.m_have_to_return_data) { - std::string return_buff; + byte_slice return_buff; const uint32_t return_code = m_config.m_pcommands_handler->invoke( m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context ); + // peer_id remains unset if dropped + if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) + m_max_packet_size = m_config.m_max_packet_size; + bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); head.m_return_code = SWAP32LE(return_code); - return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head)); - if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)})) + if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) return false; MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb @@ -577,10 +592,11 @@ public: m_cache_in_buffer.erase(sizeof(bucket_head2)); m_state = stream_state_body; m_oponent_protocol_ver = m_current_head.m_protocol_version; - if(m_current_head.m_cb > m_config.m_max_packet_size) + const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command); + if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes)) { - LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size - << ", packet header received " << m_current_head.m_cb + LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes) + << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command << ", connection will be closed."); return false; } @@ -634,6 +650,9 @@ public: boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock); + if (command == m_connection_context.handshake_command()) + m_max_packet_size = m_config.m_max_packet_size; + if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); @@ -675,6 +694,9 @@ public: boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); + if (command == m_connection_context.handshake_command()) + m_max_packet_size = m_config.m_max_packet_size; + if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 9446e3588..a1f44cab0 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -44,6 +44,7 @@ #include <boost/lambda/lambda.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/system/error_code.hpp> +#include <boost/utility/string_ref.hpp> #include <functional> #include "net/net_utils_base.h" #include "net/net_ssl.h" @@ -178,7 +179,7 @@ namespace net_utils // SSL Options if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { - if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr, timeout)) + if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout)) { if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { @@ -280,7 +281,7 @@ namespace net_utils inline - bool send(const std::string& buff, std::chrono::milliseconds timeout) + bool send(const boost::string_ref buff, std::chrono::milliseconds timeout) { try @@ -298,7 +299,7 @@ namespace net_utils // object is used as a callback and will update the ec variable when the // operation completes. The blocking_udp_client.cpp example shows how you // can use boost::bind rather than boost::lambda. - async_write(buff.c_str(), buff.size(), ec); + async_write(buff.data(), buff.size(), ec); // Block until the asynchronous operation has completed. while (ec == boost::asio::error::would_block) diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index 1d156d19c..cf637ba1d 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -131,8 +131,8 @@ namespace net_utils inline bool parse_url_ipv6(const std::string url_str, http::url_content& content) { - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 3 4 5 6 7 + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 content.port = 0; boost::smatch result; @@ -175,8 +175,8 @@ namespace net_utils ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 34 5 6 7 + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 34 5 6 7 content.port = 0; boost::smatch result; if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 643b2c486..1b1577e77 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -132,6 +132,7 @@ namespace net_utils bool handshake( boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket, boost::asio::ssl::stream_base::handshake_type type, + boost::asio::const_buffer buffer = {}, const std::string& host = {}, std::chrono::milliseconds timeout = std::chrono::seconds(15)) const; }; diff --git a/contrib/epee/include/rolling_median.h b/contrib/epee/include/rolling_median.h index 088a71d3e..877814e57 100644 --- a/contrib/epee/include/rolling_median.h +++ b/contrib/epee/include/rolling_median.h @@ -141,7 +141,6 @@ public: rolling_median_t(rolling_median_t &&m) { - free(data); memcpy(this, &m, sizeof(rolling_median_t)); m.data = NULL; } diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index a8bc945a8..c615b20e6 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -29,6 +29,7 @@ #include <boost/utility/string_ref.hpp> #include <chrono> #include <string> +#include "byte_slice.h" #include "portable_storage_template_helper.h" #include "net/http_base.h" #include "net/http_server_handlers_map2.h" @@ -74,12 +75,12 @@ namespace epee template<class t_request, class t_response, class t_transport> bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST") { - std::string req_param; - if(!serialization::store_t_to_binary(out_struct, req_param)) + byte_slice req_param; + if(!serialization::store_t_to_binary(out_struct, req_param, 16 * 1024)) return false; const http::http_response_info* pri = NULL; - if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri))) + if(!transport.invoke(uri, method, boost::string_ref{reinterpret_cast<const char*>(req_param.data()), req_param.size()}, timeout, std::addressof(pri))) { LOG_PRINT_L1("Failed to invoke http request to " << uri); return false; @@ -97,7 +98,12 @@ namespace epee return false; } - return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body)); + static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = { + 65536 * 3, // objects + 65536 * 3, // fields + 65536 * 3, // strings + }; + return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits); } template<class t_request, class t_response, class t_transport> diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index cf1262486..802e16c1b 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -27,8 +27,10 @@ #pragma once #include "portable_storage_template_helper.h" +#include <boost/utility/string_ref.hpp> #include <boost/utility/value_init.hpp> #include <functional> +#include "byte_slice.h" #include "span.h" #include "net/levin_base.h" @@ -50,6 +52,11 @@ namespace snprintf(buf, sizeof(buf), "command-%u", command); return on_levin_traffic(context, initiator, sent, error, bytes, buf); } + static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { + 8192, // objects + 16384, // fields + 16384, // strings + }; } namespace epee @@ -75,7 +82,7 @@ namespace epee return false; } serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv)) + if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) { LOG_ERROR("Failed to load_from_binary on command " << command); return false; @@ -110,18 +117,19 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - std::string buff_to_send, buff_to_recv; - stg.store_to_binary(buff_to_send); + byte_slice buff_to_send; + std::string buff_to_recv; + stg.store_to_binary(buff_to_send, 16 * 1024); on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id); + int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); return false; } typename serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv)) + if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) { on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); @@ -137,10 +145,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - std::string buff_to_send; - stg.store_to_binary(buff_to_send); + byte_slice buff_to_send; + stg.store_to_binary(buff_to_send, 16 * 1024); on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -152,7 +160,7 @@ namespace epee return false; } serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff)) + if(!stg_ret.load_from_binary(buff, &default_levin_limits)) { on_levin_traffic(context, true, false, true, buff.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); @@ -184,11 +192,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - std::string buff_to_send; + byte_slice buff_to_send; stg.store_to_binary(buff_to_send); on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id); + int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -199,10 +207,10 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff)) + if(!strg.load_from_binary(in_buff, &default_levin_limits)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in command " << command); @@ -222,7 +230,7 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out)) + if(!strg_out.store_to_binary(buff_out, 32 * 1024)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; @@ -236,7 +244,7 @@ namespace epee int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context) { serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff)) + if(!strg.load_from_binary(in_buff, &default_levin_limits)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in notify " << command); @@ -254,7 +262,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -263,13 +271,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; std::string fake_str;\ + bool handled = false; epee::byte_slice fake_str; \ return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -289,7 +297,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index 4b759a24f..f77e89cb6 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -32,7 +32,6 @@ #include "misc_language.h" #include "portable_storage_base.h" -#include "portable_storage_to_bin.h" #include "portable_storage_from_bin.h" #include "portable_storage_to_json.h" #include "portable_storage_from_json.h" @@ -42,6 +41,7 @@ namespace epee { + class byte_slice; namespace serialization { /************************************************************************/ @@ -54,6 +54,13 @@ namespace epee typedef epee::serialization::harray harray; typedef storage_entry meta_entry; + struct limits_t + { + size_t n_objects; + size_t n_fields; + size_t n_strings; // not counting field names + }; + portable_storage(){} virtual ~portable_storage(){} hsection open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist = false); @@ -83,9 +90,9 @@ namespace epee bool delete_entry(const std::string& pentry_name, hsection hparent_section = nullptr); //------------------------------------------------------------------------------- - bool store_to_binary(binarybuffer& target); - bool load_from_binary(const epee::span<const uint8_t> target); - bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); } + bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); + bool load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); } template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -133,24 +140,8 @@ namespace epee { return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format } - - inline - bool portable_storage::store_to_binary(binarybuffer& target) - { - TRY_ENTRY(); - std::stringstream ss; - storage_block_header sbh = AUTO_VAL_INIT(sbh); - sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); - sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); - sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; - ss.write((const char*)&sbh, sizeof(storage_block_header)); - pack_entry_to_buff(ss, m_root); - target = ss.str(); - return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) - } inline - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source) + bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) { m_root.m_entries.clear(); if(source.size() < sizeof(storage_block_header)) @@ -173,6 +164,8 @@ namespace epee } TRY_ENTRY(); throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); + if (limits) + buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); buf_reader.read(m_root); return true;//TODO: CATCH_ENTRY("portable_storage::load_from_binary", false); @@ -282,6 +275,7 @@ namespace epee static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value"); TRY_ENTRY(); CHECK_AND_ASSERT(psection, nullptr); + CHECK_AND_ASSERT(!pentry_name.empty(), nullptr); auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry)); return &ins_res.first->second; CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr); diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index b39dc7c92..9e7b6ec34 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -29,6 +29,7 @@ #pragma once #include "misc_language.h" +#include "misc_log_ex.h" #include "portable_storage_base.h" #include "portable_storage_bin_utils.h" @@ -42,6 +43,24 @@ namespace epee { namespace serialization { + template<typename T> + struct ps_min_bytes { + static constexpr const size_t strict = 4096; // actual low bound + }; + template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; }; + template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; }; + template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; }; + struct throwable_buffer_reader { throwable_buffer_reader(const void* ptr, size_t sz); @@ -61,6 +80,9 @@ namespace epee void read(section& sec); void read(std::string& str); void read(array_entry &ae); + template<class t_type> + size_t min_bytes() const; + void set_limits(size_t objects, size_t fields, size_t strings); private: struct recursuion_limitation_guard { @@ -81,6 +103,13 @@ namespace epee const uint8_t* m_ptr; size_t m_count; size_t m_recursion_count; + size_t m_objects; + size_t m_fields; + size_t m_strings; + + size_t max_objects; + size_t max_fields; + size_t max_strings; }; inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz) @@ -92,6 +121,12 @@ namespace epee m_ptr = (uint8_t*)ptr; m_count = sz; m_recursion_count = 0; + m_objects = 0; + m_fields = 0; + m_strings = 0; + max_objects = std::numeric_limits<size_t>::max(); + max_fields = std::numeric_limits<size_t>::max(); + max_strings = std::numeric_limits<size_t>::max(); } inline void throwable_buffer_reader::read(void* target, size_t count) @@ -108,6 +143,7 @@ namespace epee RECURSION_LIMITATION(); uint8_t name_len = 0; read(name_len); + CHECK_AND_ASSERT_THROW_MES(name_len > 0, "Section name is missing"); sce_name.resize(name_len); read((void*)sce_name.data(), name_len); } @@ -138,7 +174,18 @@ namespace epee //for pod types array_entry_t<type_name> sa; size_t size = read_varint(); - CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed"); + CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed"); + if (std::is_same<type_name, section>()) + { + CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects"); + m_objects += size; + } + else if (std::is_same<type_name, std::string>()) + { + CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings"); + m_strings += size; + } + sa.reserve(size); //TODO: add some optimization here later while(size--) @@ -204,6 +251,8 @@ namespace epee inline storage_entry throwable_buffer_reader::read_se<std::string>() { RECURSION_LIMITATION(); + CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings"); + m_strings += 1; return storage_entry(read<std::string>()); } @@ -212,6 +261,8 @@ namespace epee inline storage_entry throwable_buffer_reader::read_se<section>() { RECURSION_LIMITATION(); + CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects"); + ++m_objects; section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio storage_entry se(std::move(s)); section& section_entry = boost::get<section>(se); @@ -263,12 +314,16 @@ namespace epee RECURSION_LIMITATION(); sec.m_entries.clear(); size_t count = read_varint(); + CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields"); + m_fields += count; while(count--) { //read section name string std::string sec_name; read_sec_name(sec_name); - sec.m_entries.emplace(std::move(sec_name), load_storage_entry()); + const auto insert_loc = sec.m_entries.lower_bound(sec_name); + CHECK_AND_ASSERT_THROW_MES(insert_loc == sec.m_entries.end() || insert_loc->first != sec_name, "duplicate key: " << sec_name); + sec.m_entries.emplace_hint(insert_loc, std::move(sec_name), load_storage_entry()); } } inline @@ -289,5 +344,12 @@ namespace epee RECURSION_LIMITATION(); CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported"); } + inline + void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings) + { + max_objects = objects; + max_fields = fields; + max_strings = strings; + } } } diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 13c870d44..16dd565ec 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -28,6 +28,7 @@ #include <string> +#include "byte_slice.h" #include "parserse_base_utils.h" #include "portable_storage.h" #include "file_io_utils.h" @@ -84,10 +85,10 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff) + bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL) { portable_storage ps; - bool rs = ps.load_from_binary(binary_buff); + bool rs = ps.load_from_binary(binary_buff, limits); if(!rs) return false; @@ -111,18 +112,18 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - bool store_t_to_binary(t_struct& str_in, std::string& binary_buff, size_t indent = 0) + bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff, size_t initial_buffer_size = 8192) { portable_storage ps; str_in.store(ps); - return ps.store_to_binary(binary_buff); + return ps.store_to_binary(binary_buff, initial_buffer_size); } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - std::string store_t_to_binary(t_struct& str_in, size_t indent = 0) + byte_slice store_t_to_binary(t_struct& str_in, size_t initial_buffer_size = 8192) { - std::string binary_buff; - store_t_to_binary(str_in, binary_buff, indent); + byte_slice binary_buff; + store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } } diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 137497e19..49a7be185 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -211,6 +211,7 @@ namespace epee for(const section_pair& se: sec.m_entries) { CHECK_AND_ASSERT_THROW_MES(se.first.size() < std::numeric_limits<uint8_t>::max(), "storage_entry_name is too long: " << se.first.size() << ", val: " << se.first); + CHECK_AND_ASSERT_THROW_MES(!se.first.empty(), "storage_entry_name is empty"); uint8_t len = static_cast<uint8_t>(se.first.size()); strm.write((const char*)&len, sizeof(len)); strm.write(se.first.data(), size_t(len)); |