diff options
Diffstat (limited to 'contrib')
40 files changed, 1703 insertions, 174 deletions
diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 6d369d4d8..4ea3fa54b 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -27,6 +27,7 @@ #pragma once #include "misc_log_ex.h" +#include "string_tools.h" #include <atomic> #include <condition_variable> #include <functional> @@ -37,6 +38,8 @@ #include <stdio.h> #endif #include <boost/thread.hpp> +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> #ifdef HAVE_READLINE #include "readline_buffer.h" diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index c387743a6..989d16fba 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -29,7 +29,8 @@ #define _FILE_IO_UTILS_H_ #include <iostream> -#include <boost/filesystem.hpp> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> namespace epee { @@ -63,7 +64,7 @@ namespace file_io_utils } inline - bool get_file_time(const std::string& path_to_file, OUT time_t& ft) + bool get_file_time(const std::string& path_to_file, time_t& ft) { boost::system::error_code ec; ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); @@ -132,6 +133,26 @@ namespace file_io_utils return false; } } + + inline + bool get_file_size(const std::string& path_to_file, uint64_t &size) + { + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + size = fstream.tellg(); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } + } + } } diff --git a/contrib/epee/include/math_helper.h b/contrib/epee/include/math_helper.h index 90398acbb..ef839f609 100644 --- a/contrib/epee/include/math_helper.h +++ b/contrib/epee/include/math_helper.h @@ -37,6 +37,7 @@ #include <boost/uuid/random_generator.hpp> #include "misc_os_dependent.h" +#include "syncobj.h" namespace epee { diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 67fd93206..35ec0950b 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -28,26 +28,7 @@ #ifndef _MISC_LOG_EX_H_ #define _MISC_LOG_EX_H_ -#include "static_initializer.h" -#include "string_tools.h" -#include "time_helper.h" -#include "misc_os_dependent.h" - -#include "syncobj.h" - -#include <iostream> -#include <sstream> -#include <iomanip> -#include <fstream> -#include <algorithm> -#include <list> -#include <map> #include <string> -#include <time.h> -#include <boost/cstdint.hpp> -#include <boost/thread.hpp> -#include <boost/filesystem.hpp> -#include <boost/algorithm/string.hpp> #include "easylogging++.h" diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 69ded09e5..99690b301 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -23,6 +23,10 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +#ifdef _WIN32 +#include <Winsock2.h> +#endif + #ifdef WIN32 #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -42,6 +46,9 @@ #include <mach/mach.h> #endif +#include <iostream> +#include <boost/lexical_cast.hpp> + #pragma once namespace epee { diff --git a/contrib/epee/include/net/abstract_tcp_server.h b/contrib/epee/include/net/abstract_tcp_server.h index 000305cfa..cbad1717c 100644 --- a/contrib/epee/include/net/abstract_tcp_server.h +++ b/contrib/epee/include/net/abstract_tcp_server.h @@ -305,7 +305,7 @@ namespace net_utils m_connections.back().powner = this; m_connections.back().m_self_it = --m_connections.end(); m_connections.back().m_context.m_remote_address = remote_address; - m_connections.back().m_htread = threads_helper::create_thread(ConnectionHandlerProc, &m_connections.back()); + m_connections.back().m_htread = threads_helper::create_thread(ConnectionHandlerProc, &m_connections.back()); // ugh, seems very risky return true; } diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 33fec8ec5..c0401c8b0 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -54,8 +54,8 @@ #include <boost/thread/thread.hpp> #include "net_utils_base.h" #include "syncobj.h" -#include "../../../../src/p2p/connection_basic.hpp" -#include "../../../../src/p2p/network_throttle-detail.hpp" +#include "connection_basic.hpp" +#include "network_throttle-detail.hpp" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 00d03567c..870f6c2b2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -286,7 +286,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); - epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred * 1024); + epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred); } double delay=0; // will be calculated - how much we should sleep to obey speed limit etc @@ -297,7 +297,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { { //_scope_dbg1("CRITICAL_REGION_LOCAL"); CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); - delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); // decission from global throttle + delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); } delay *= 0.5; @@ -482,9 +482,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //some data should be wrote to stream //request complete - if (speed_limit_is_enabled()) { - sleep_before_packet(cb, 1, 1); - } + // No sleeping here; sleeping is done once and for all in "handle_write" m_send_que_lock.lock(); // *** critical *** epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_send_que_lock.unlock();}); @@ -607,6 +605,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } logger_handle_net_write(cb); + // The single sleeping that is needed for correctly handling "out" speed throttling if (speed_limit_is_enabled()) { sleep_before_packet(cb, 1, 1); } @@ -736,7 +735,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::asio::placeholders::error)); return true; - CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::init_server", false); + } + catch (const std::exception &e) + { + MFATAL("Error starting server: " << e.what()); + return false; + } + catch (...) + { + MFATAL("Error starting server"); + return false; + } } //----------------------------------------------------------------------------- PUSH_WARNINGS diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp new file mode 100644 index 000000000..16de469a7 --- /dev/null +++ b/contrib/epee/include/net/connection_basic.hpp @@ -0,0 +1,141 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// ! This file might contain variable names same as in template class connection<> +// ! from files contrib/epee/include/net/abstract_tcp_server2.* +// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;) +// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) +// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#ifndef INCLUDED_p2p_connection_basic_hpp +#define INCLUDED_p2p_connection_basic_hpp + + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "net/net_utils_base.h" +#include "syncobj.h" + +namespace epee +{ +namespace net_utils +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + /// Represents a single connection from a client. + +class connection_basic_pimpl; // PIMPL for this class + + enum t_connection_type { // type of the connection (of this server), e.g. so that we will know how to limit it + e_connection_type_NET = 0, // default (not used?) + e_connection_type_RPC = 1, // the rpc commands (probably not rate limited, not chunked, etc) + e_connection_type_P2P = 2 // to other p2p node (probably limited) + }; + + std::string to_string(t_connection_type type); + +class connection_basic { // not-templated base class for rapid developmet of some code parts + public: + std::unique_ptr< connection_basic_pimpl > mI; // my Implementation + + // moved here from orginal connecton<> - common member variables that do not depend on template in connection<> + volatile uint32_t m_want_close_connection; + std::atomic<bool> m_was_shutdown; + critical_section m_send_que_lock; + std::list<std::string> m_send_que; + volatile bool m_is_multithreaded; + double m_start_time; + /// Strand to ensure the connection's handlers are not called concurrently. + boost::asio::io_service::strand strand_; + /// Socket for the connection. + boost::asio::ip::tcp::socket socket_; + + std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/-- + public: + // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator + connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number); + + virtual ~connection_basic() noexcept(false); + + // various handlers to be called from connection class: + void do_send_handler_write(const void * ptr , size_t cb); + void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part + + void logger_handle_net_write(size_t size); // network data written + void logger_handle_net_read(size_t size); // network data read + + void set_start_time(); + + // config for rate limit + + static void set_rate_up_limit(uint64_t limit); + static void set_rate_down_limit(uint64_t limit); + static uint64_t get_rate_up_limit(); + static uint64_t get_rate_down_limit(); + + // config misc + static void set_tos_flag(int tos); // ToS / QoS flag + static int get_tos_flag(); + + // handlers and sleep + void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) + static void save_limit_to_file(int limit); ///< for dr-monero + static double get_sleep_time(size_t cb); + + static void set_save_graph(bool save_graph); +}; + +} // nameserver +} // nameserver + +#endif + + diff --git a/contrib/epee/include/net/http_auth.h b/contrib/epee/include/net/http_auth.h index bf368e6f4..841cebc17 100644 --- a/contrib/epee/include/net/http_auth.h +++ b/contrib/epee/include/net/http_auth.h @@ -33,7 +33,7 @@ #include <functional> #include <string> #include <utility> - +#include "wipeable_string.h" #include "http_base.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -48,12 +48,12 @@ namespace net_utils struct login { login() : username(), password() {} - login(std::string username_, std::string password_) + login(std::string username_, wipeable_string password_) : username(std::move(username_)), password(std::move(password_)) {} std::string username; - std::string password; + wipeable_string password; }; //! Implements RFC 2617 digest auth. Digests from RFC 7616 can be added. diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index d77c84c28..1edf65928 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -27,6 +27,7 @@ #pragma once +#include <ctype.h> #include <boost/shared_ptr.hpp> #include <boost/regex.hpp> #include <boost/lexical_cast.hpp> @@ -50,6 +51,7 @@ #include "http_auth.h" #include "to_nonconst_iterator.h" #include "net_parse_helpers.h" +#include "syncobj.h" //#include "shlwapi.h" @@ -235,7 +237,8 @@ namespace net_utils namespace http { - class http_simple_client: public i_target_handler + template<typename net_client_type> + class http_simple_client_template: public i_target_handler { private: enum reciev_machine_state @@ -258,7 +261,7 @@ namespace net_utils }; - blocked_mode_client m_net_client; + net_client_type m_net_client; std::string m_host_buff; std::string m_port; http_client_auth m_auth; @@ -275,7 +278,7 @@ namespace net_utils bool m_ssl; public: - explicit http_simple_client() + explicit http_simple_client_template() : i_target_handler() , m_net_client() , m_host_buff() @@ -427,6 +430,15 @@ namespace net_utils CRITICAL_REGION_LOCAL(m_lock); return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params); } + //--------------------------------------------------------------------------- + bool test(const std::string &s, std::chrono::milliseconds timeout) // TEST FUNC ONLY + { + CRITICAL_REGION_LOCAL(m_lock); + m_net_client.set_test_data(s); + m_state = reciev_machine_state_header; + return handle_reciev(timeout); + } + //--------------------------------------------------------------------------- private: //--------------------------------------------------------------------------- inline bool handle_reciev(std::chrono::milliseconds timeout) @@ -741,87 +753,107 @@ namespace net_utils return true; } //--------------------------------------------------------------------------- - inline - bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process) - { + inline bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process) + { MTRACE("http_stream_filter::parse_cached_header(*)"); - - STATIC_REGEXP_EXPR_1(rexp_mach_field, - "\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)|(Origin)" - // 12 3 4 5 6 7 8 9 10 11 - "|([\\w-]+?)) ?: ?((.*?)(\r?\n))[^\t ]", - //12 13 14 15 - boost::regex::icase | boost::regex::normal); - boost::smatch result; - std::string::const_iterator it_current_bound = m_cache_to_process.begin(); - std::string::const_iterator it_end_bound = m_cache_to_process.end(); - - - - //lookup all fields and fill well-known fields - while( boost::regex_search( it_current_bound, it_end_bound, result, rexp_mach_field, boost::match_default) && result[0].matched) + const char *ptr = m_cache_to_process.c_str(); + while (ptr[0] != '\r' || ptr[1] != '\n') { - const size_t field_val = 14; - //const size_t field_etc_name = 11; - - int i = 2; //start position = 2 - if(result[i++].matched)//"Connection" - body_info.m_connection = result[field_val]; - else if(result[i++].matched)//"Referrer" - body_info.m_referer = result[field_val]; - else if(result[i++].matched)//"Content-Length" - body_info.m_content_length = result[field_val]; - else if(result[i++].matched)//"Content-Type" - body_info.m_content_type = result[field_val]; - else if(result[i++].matched)//"Transfer-Encoding" - body_info.m_transfer_encoding = result[field_val]; - else if(result[i++].matched)//"Content-Encoding" - body_info.m_content_encoding = result[field_val]; - else if(result[i++].matched)//"Host" - { body_info.m_host = result[field_val]; - string_tools::trim(body_info.m_host); + // optional \n + if (*ptr == '\n') + ++ptr; + // an identifier composed of letters or - + const char *key_pos = ptr; + while (isalnum(*ptr) || *ptr == '_' || *ptr == '-') + ++ptr; + const char *key_end = ptr; + // optional space (not in RFC, but in previous code) + if (*ptr == ' ') + ++ptr; + CHECK_AND_ASSERT_MES(*ptr == ':', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); + ++ptr; + // optional whitespace, but not newlines - line folding is obsolete, let's ignore it + while (isblank(*ptr)) + ++ptr; + const char *value_pos = ptr; + while (*ptr != '\r' && *ptr != '\n') + ++ptr; + const char *value_end = ptr; + // optional trailing whitespace + while (value_end > value_pos && isblank(*(value_end-1))) + --value_end; + if (*ptr == '\r') + ++ptr; + CHECK_AND_ASSERT_MES(*ptr == '\n', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); + ++ptr; + + const std::string key = std::string(key_pos, key_end - key_pos); + const std::string value = std::string(value_pos, value_end - value_pos); + if (!key.empty()) + { + if (!string_tools::compare_no_case(key, "Connection")) + body_info.m_connection = value; + else if(!string_tools::compare_no_case(key, "Referrer")) + body_info.m_referer = value; + else if(!string_tools::compare_no_case(key, "Content-Length")) + body_info.m_content_length = value; + else if(!string_tools::compare_no_case(key, "Content-Type")) + body_info.m_content_type = value; + else if(!string_tools::compare_no_case(key, "Transfer-Encoding")) + body_info.m_transfer_encoding = value; + else if(!string_tools::compare_no_case(key, "Content-Encoding")) + body_info.m_content_encoding = value; + else if(!string_tools::compare_no_case(key, "Host")) + body_info.m_host = value; + else if(!string_tools::compare_no_case(key, "Cookie")) + body_info.m_cookie = value; + else if(!string_tools::compare_no_case(key, "User-Agent")) + body_info.m_user_agent = value; + else if(!string_tools::compare_no_case(key, "Origin")) + body_info.m_origin = value; + else + body_info.m_etc_fields.emplace_back(key, value); } - else if(result[i++].matched)//"Cookie" - body_info.m_cookie = result[field_val]; - else if(result[i++].matched)//"User-Agent" - body_info.m_user_agent = result[field_val]; - else if(result[i++].matched)//"Origin" - body_info.m_origin = result[field_val]; - else if(result[i++].matched)//e.t.c (HAVE TO BE MATCHED!) - body_info.m_etc_fields.emplace_back(result[12], result[field_val]); - else - {CHECK_AND_ASSERT_MES(false, false, "http_stream_filter::parse_cached_header() not matched last entry in:"<<m_cache_to_process);} - - it_current_bound = result[(int)result.size()-1]. first; } - return true; - + return true; } - inline - bool analize_first_response_line() + //--------------------------------------------------------------------------- + inline bool analize_first_response_line() { - - //First line response, look like this: "HTTP/1.1 200 OK" - STATIC_REGEXP_EXPR_1(rexp_match_first_response_line, "^HTTP/(\\d+).(\\d+) ((\\d)\\d{2})( [^\n]*)?\r?\n", boost::regex::icase | boost::regex::normal); - // 1 2 34 5 - //size_t match_len = 0; - boost::smatch result; - if(boost::regex_search( m_header_cache, result, rexp_match_first_response_line, boost::match_default) && result[0].matched) - { - CHECK_AND_ASSERT_MES(result[1].matched&&result[2].matched, false, "http_stream_filter::handle_invoke_reply_line() assert failed..."); - m_response_info.m_http_ver_hi = boost::lexical_cast<int>(result[1]); - m_response_info.m_http_ver_lo = boost::lexical_cast<int>(result[2]); - m_response_info.m_response_code = boost::lexical_cast<int>(result[3]); - - m_header_cache.erase(to_nonsonst_iterator(m_header_cache, result[0].first), to_nonsonst_iterator(m_header_cache, result[0].second)); - return true; - }else - { - LOG_ERROR("http_stream_filter::handle_invoke_reply_line(): Failed to match first response line:" << m_header_cache); - return false; - } - + //First line response, look like this: "HTTP/1.1 200 OK" + const char *ptr = m_header_cache.c_str(); + CHECK_AND_ASSERT_MES(!memcmp(ptr, "HTTP/", 5), false, "Invalid first response line: " + m_header_cache); + ptr += 5; + CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + unsigned long ul; + char *end; + ul = strtoul(ptr, &end, 10); + CHECK_AND_ASSERT_MES(ul <= INT_MAX && *end =='.', false, "Invalid first response line: " + m_header_cache); + m_response_info.m_http_ver_hi = ul; + ptr = end + 1; + CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + ul = strtoul(ptr, &end, 10); + CHECK_AND_ASSERT_MES(ul <= INT_MAX && isblank(*end), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + m_response_info.m_http_ver_lo = ul; + ptr = end + 1; + while (isblank(*ptr)) + ++ptr; + CHECK_AND_ASSERT_MES(isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + ul = strtoul(ptr, &end, 10); + CHECK_AND_ASSERT_MES(ul >= 100 && ul <= 999 && isspace(*end), false, "Invalid first response line: " + m_header_cache); + m_response_info.m_response_code = ul; + ptr = end; + // ignore the optional text, till the end + while (*ptr != '\r' && *ptr != '\n') + ++ptr; + if (*ptr == '\r') + ++ptr; + CHECK_AND_ASSERT_MES(*ptr == '\n', false, "Invalid first response line: " << m_header_cache); + ++ptr; + + m_header_cache.erase(0, ptr - m_header_cache.c_str()); + return true; } inline bool set_reply_content_encoder() @@ -956,6 +988,7 @@ namespace net_utils return true; } }; + typedef http_simple_client_template<blocked_mode_client> http_simple_client; } } } diff --git a/contrib/epee/include/net/http_client_base.h b/contrib/epee/include/net/http_client_base.h index f5fb57d03..c3da28718 100644 --- a/contrib/epee/include/net/http_client_base.h +++ b/contrib/epee/include/net/http_client_base.h @@ -38,8 +38,8 @@ namespace epee virtual ~i_sub_handler(){} virtual bool update_in( std::string& piece_of_transfer)=0; - virtual void stop(std::string& OUT collect_remains)=0; - virtual bool update_and_stop(std::string& OUT collect_remains, bool& is_changed) + virtual void stop(std::string& collect_remains)=0; + virtual bool update_and_stop(std::string& collect_remains, bool& is_changed) { is_changed = true; bool res = this->update_in(collect_remains); @@ -66,7 +66,7 @@ namespace epee { return m_powner_filter->handle_target_data(piece_of_transfer); } - virtual void stop(std::string& OUT collect_remains) + virtual void stop(std::string& collect_remains) { } diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index c555707ac..c18f7f706 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -345,7 +345,12 @@ namespace net_utils { analize_http_method(result, m_query_info.m_http_method, m_query_info.m_http_ver_hi, m_query_info.m_http_ver_hi); m_query_info.m_URI = result[10]; - parse_uri(m_query_info.m_URI, m_query_info.m_uri_content); + if (!parse_uri(m_query_info.m_URI, m_query_info.m_uri_content)) + { + m_state = http_state_error; + MERROR("Failed to parse URI: m_query_info.m_URI"); + return false; + } m_query_info.m_http_method_str = result[2]; m_query_info.m_full_request_str = result[0]; diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 2e320ac8d..65705351d 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -35,6 +35,8 @@ #include "levin_base.h" #include "misc_language.h" +#include "syncobj.h" +#include "misc_os_dependent.h" #include <random> #include <chrono> @@ -97,6 +99,7 @@ public: async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE) {} + ~async_protocol_handler_config() { set_handler(NULL, NULL); } void del_out_connections(size_t count); }; diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index ee0e13fc1..2c2efcd82 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -31,23 +31,16 @@ //#include <Winsock2.h> //#include <Ws2tcpip.h> -#include <boost/lexical_cast.hpp> -#include <iostream> -#include <istream> -#include <ostream> #include <string> #include <boost/version.hpp> #include <boost/asio.hpp> #include <boost/asio/ssl.hpp> #include <boost/asio/steady_timer.hpp> -#include <boost/preprocessor/selection/min.hpp> #include <boost/lambda/bind.hpp> #include <boost/lambda/lambda.hpp> #include <boost/interprocess/detail/atomic.hpp> #include "net/net_utils_base.h" #include "misc_language.h" -//#include "profile_tools.h" -#include "../string_tools.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index 08d2a2000..708cce0ff 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -103,7 +103,7 @@ namespace net_utils STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); boost::smatch result; - if(!boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched) + if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) { LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); content.m_path = uri; @@ -139,7 +139,7 @@ namespace net_utils // 12 34 5 6 7 content.port = 0; boost::smatch result; - if(!boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched) + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) { LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); //content.m_path = uri; diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 04e3fe6a4..7615786be 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -29,13 +29,11 @@ #ifndef _NET_UTILS_BASE_H_ #define _NET_UTILS_BASE_H_ -#include <boost/asio/io_service.hpp> #include <boost/uuid/uuid.hpp> -#include <memory> +#include <boost/asio/io_service.hpp> #include <typeinfo> #include <type_traits> #include "serialization/keyvalue_serialization.h" -#include "string_tools.h" #include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -45,7 +43,6 @@ #define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) #endif - namespace epee { namespace net_utils @@ -214,19 +211,8 @@ namespace net_utils inline bool operator>=(const network_address& lhs, const network_address& rhs) { return !lhs.less(rhs); } - inline bool create_network_address(network_address &address, const std::string &string, uint16_t default_port = 0) - { - uint32_t ip; - uint16_t port; - if (epee::string_tools::parse_peer_from_string(ip, port, string)) - { - if (default_port && !port) - port = default_port; - address = ipv4_network_address{ip, port}; - return true; - } - return false; - } + bool create_network_address(network_address &address, const std::string &string, uint16_t default_port = 0); + /************************************************************************/ /* */ /************************************************************************/ @@ -309,21 +295,8 @@ namespace net_utils //some helpers - inline - std::string print_connection_context(const connection_context_base& ctx) - { - std::stringstream ss; - ss << ctx.m_remote_address.str() << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT"); - return ss.str(); - } - - inline - std::string print_connection_context_short(const connection_context_base& ctx) - { - std::stringstream ss; - ss << ctx.m_remote_address.str() << (ctx.m_is_income ? " INC":" OUT"); - return ss.str(); - } + std::string print_connection_context(const connection_context_base& ctx); + std::string print_connection_context_short(const connection_context_base& ctx); inline MAKE_LOGGABLE(connection_context_base, ct, os) { diff --git a/contrib/epee/include/net/network_throttle-detail.hpp b/contrib/epee/include/net/network_throttle-detail.hpp new file mode 100644 index 000000000..dba15a5ed --- /dev/null +++ b/contrib/epee/include/net/network_throttle-detail.hpp @@ -0,0 +1,125 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: throttle details, implementing rate limiting */ + + +#ifndef INCLUDED_throttle_detail_hpp +#define INCLUDED_throttle_detail_hpp + +#include "network_throttle.hpp" + +namespace epee +{ +namespace net_utils +{ + + +class network_throttle : public i_network_throttle { + private: + struct packet_info { + size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second) + packet_info(); + }; + + + network_speed_bps m_target_speed; + size_t m_network_add_cost; // estimated add cost of headers + size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to + size_t m_network_max_segment; // recommended max size of 1 TCP transmission + + const size_t m_window_size; // the number of samples to average over + network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot() + // TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements + + std::vector< packet_info > m_history; // the history of bw usage + network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer + network_time_seconds m_start_time; // when we were created + bool m_any_packet_yet; // did we yet got any packet to count + + std::string m_name; // my name for debug and logs + std::string m_nameshort; // my name for debug and logs (used in log file name) + + // each sample is now 1 second + public: + network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1); + virtual ~network_throttle(); + virtual void set_name(const std::string &name); + virtual void set_target_speed( network_speed_kbps target ); + virtual network_speed_kbps get_target_speed(); + + // add information about events: + virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc + + virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc) + + virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic + + // time calculations: + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info) + + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size + virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO + + virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported + virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame + virtual double get_current_speed() const; + + private: + virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 + virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size); + virtual void logger_handle_net(const std::string &filename, double time, size_t size); +}; + +/*** + * The complete set of traffic throttle for one typical connection +*/ +struct network_throttle_bw { + public: + network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually) + network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually) + network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually) + + public: + network_throttle_bw(const std::string &name1); +}; + + + +} // namespace net_utils +} // namespace epee + + +#endif + + diff --git a/contrib/epee/include/net/network_throttle.hpp b/contrib/epee/include/net/network_throttle.hpp new file mode 100644 index 000000000..fffd22a6a --- /dev/null +++ b/contrib/epee/include/net/network_throttle.hpp @@ -0,0 +1,174 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief interface for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: throttle basic interface */ +/* rfree: also includes the manager for singeton/global such objects */ + + +#ifndef INCLUDED_network_throttle_hpp +#define INCLUDED_network_throttle_hpp + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include "syncobj.h" + +#include "net/net_utils_base.h" +#include "misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + +#include <memory> +#include <mutex> +#include <fstream> + +namespace epee +{ +namespace net_utils +{ + +// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono? +typedef double network_speed_kbps; // externally, for parameters and return values, all defined in kilobytes per second +typedef double network_speed_bps; // throttle-internally, bytes per second +typedef double network_time_seconds; +typedef double network_MB; + +class i_network_throttle; + +/*** +@brief All information about given throttle - speed calculations +*/ +struct calculate_times_struct { + double average; + double window; + double delay; + double recomendetDataSize; +}; +typedef calculate_times_struct calculate_times_struct; + + +namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working + +/*** +@brief Access to simple throttles, with singlton to access global network limits +*/ +class network_throttle_manager { + // provides global (singleton) in/inreq/out throttle access + + // [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/ + // [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests) + + //protected: + public: // XXX + + static boost::mutex m_lock_get_global_throttle_in; + static boost::mutex m_lock_get_global_throttle_inreq; + static boost::mutex m_lock_get_global_throttle_out; + + friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic_pimpl; // ditto + + public: + static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in + static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously + static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously +}; + + + +/*** +@brief interface for the throttle, see the derivated class +*/ +class i_network_throttle { + public: + virtual void set_name(const std::string &name)=0; + virtual void set_target_speed( network_speed_kbps target )=0; + virtual network_speed_kbps get_target_speed()=0; + + virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc + virtual void tick() =0; // poke and update timers/history + + // time calculations: + + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate: + // Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size + + // Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now + + virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer + + virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future + + virtual double get_time_seconds() const =0; // a timer + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + + +}; + + +// ... more in the -advanced.h file + + +} // namespace net_utils +} // namespace epee + + +#endif + + + diff --git a/contrib/epee/include/profile_tools.h b/contrib/epee/include/profile_tools.h index f285fe48b..a0b5f77f4 100644 --- a/contrib/epee/include/profile_tools.h +++ b/contrib/epee/include/profile_tools.h @@ -28,6 +28,8 @@ #ifndef _PROFILE_TOOLS_H_ #define _PROFILE_TOOLS_H_ +#include "misc_os_dependent.h" + namespace epee { diff --git a/contrib/epee/include/readline_buffer.h b/contrib/epee/include/readline_buffer.h index cda7e34f9..87c8826cb 100644 --- a/contrib/epee/include/readline_buffer.h +++ b/contrib/epee/include/readline_buffer.h @@ -2,9 +2,7 @@ #include <streambuf> #include <sstream> -#include <iostream> #include <vector> -#include <algorithm> namespace rdln { diff --git a/contrib/epee/include/reg_exp_definer.h b/contrib/epee/include/reg_exp_definer.h index e2bed5c3f..eb11c9e10 100644 --- a/contrib/epee/include/reg_exp_definer.h +++ b/contrib/epee/include/reg_exp_definer.h @@ -29,7 +29,7 @@ #define _REG_EXP_DEFINER_H_ #include <boost/interprocess/detail/atomic.hpp> - +#include "syncobj.h" namespace epee { diff --git a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h index 2e020b136..7087136cc 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h +++ b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h @@ -26,6 +26,13 @@ #pragma once +#include <set> +#include <list> +#include <vector> +#include <deque> +#include <boost/mpl/vector.hpp> +#include <boost/mpl/contains_fwd.hpp> + namespace epee { namespace serialization diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index ea4ba63dd..5660c09d1 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -108,7 +108,7 @@ namespace epee template<typename T> constexpr bool has_padding() noexcept { - return !std::is_pod<T>() || alignof(T) != 1; + return !std::is_pod<T>::value || alignof(T) != 1; } //! \return Cast data from `src` as `span<const std::uint8_t>`. diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 04b57376c..727f36552 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -25,6 +25,8 @@ // #pragma once +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "parserse_base_utils.h" #include "file_io_utils.h" diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 38e9f9098..9501bbc2a 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -28,6 +28,7 @@ #pragma once +#include "pragma_comp_defs.h" #include "misc_language.h" #include "portable_storage_base.h" diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index f4a16cfae..52aa09eba 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -28,7 +28,7 @@ #pragma once -#include <regex> +#include <boost/regex.hpp> #include "misc_language.h" #include "portable_storage_base.h" @@ -146,7 +146,7 @@ POP_WARNINGS if(std::all_of(from.begin(), from.end(), ::isdigit)) to = boost::lexical_cast<uint64_t>(from); // MyMonero ISO 8061 timestamp (2017-05-06T16:27:06Z) - else if (std::regex_match (from, std::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ"))) + else if (boost::regex_match (from, boost::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ"))) { // Convert to unix timestamp std::tm tm = {}; diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 25639263c..5a1ef0743 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -35,15 +35,15 @@ # include <windows.h> #endif +#include <string.h> #include <locale> #include <cstdlib> #include <string> #include <type_traits> -#include <regex> #include <boost/uuid/uuid.hpp> #include <boost/uuid/uuid_io.hpp> #include <boost/lexical_cast.hpp> -#include <boost/algorithm/string.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "hex.h" #include "span.h" #include "warnings.h" @@ -161,7 +161,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized) val = boost::lexical_cast<XType>(str_id); return true; } - catch(std::exception& /*e*/) + catch(const std::exception& /*e*/) { //const char* pmsg = e.what(); return false; @@ -350,11 +350,8 @@ POP_WARNINGS s = *(t_pod_type*)bin_buff.data(); return true; } - //---------------------------------------------------------------------------- - inline bool validate_hex(uint64_t length, const std::string& str) - { - return std::regex_match(str, std::regex("'^[0-9abcdefABCDEF]+$'")) && str.size() == length; - } + //---------------------------------------------------------------------------- + bool validate_hex(uint64_t length, const std::string& str); //---------------------------------------------------------------------------- inline std::string get_extension(const std::string& str) { diff --git a/contrib/epee/include/wipeable_string.h b/contrib/epee/include/wipeable_string.h new file mode 100644 index 000000000..66d3e8e2b --- /dev/null +++ b/contrib/epee/include/wipeable_string.h @@ -0,0 +1,70 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stddef.h> +#include <vector> +#include <string> + +namespace epee +{ + class wipeable_string + { + public: + wipeable_string() {} + wipeable_string(const wipeable_string &other); + wipeable_string(wipeable_string &&other); + wipeable_string(const std::string &other); + wipeable_string(std::string &&other); + wipeable_string(const char *s); + ~wipeable_string(); + void wipe(); + void push_back(char c); + void pop_back(); + const char *data() const noexcept { return buffer.data(); } + size_t size() const noexcept { return buffer.size(); } + bool empty() const noexcept { return buffer.empty(); } + void resize(size_t sz); + void reserve(size_t sz); + void clear(); + bool operator==(const wipeable_string &other) const noexcept { return buffer == other.buffer; } + bool operator!=(const wipeable_string &other) const noexcept { return buffer != other.buffer; } + wipeable_string &operator=(wipeable_string &&other); + wipeable_string &operator=(const wipeable_string &other); + + static void set_wipe(void *(*f)(void*, size_t)) { wipefunc = f; } + + private: + void grow(size_t sz, size_t reserved = 0); + + private: + std::vector<char> buffer; + static void *(*wipefunc)(void*, size_t); + }; +} diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index bd6714791..b6967e8fc 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,7 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp) +add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp + connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp) if (USE_READLINE AND GNU_READLINE_FOUND) add_library(epee_readline STATIC readline_buffer.cpp) endif() diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp new file mode 100644 index 000000000..534044a79 --- /dev/null +++ b/contrib/epee/src/connection_basic.cpp @@ -0,0 +1,290 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#include "net/connection_basic.hpp" + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "net/net_utils_base.h" +#include "misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include <boost/filesystem.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <fstream> +#include <sstream> +#include <iomanip> +#include <algorithm> +#include <mutex> + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> +#include "net/abstract_tcp_server2.h" + +// TODO: +#include "net/network_throttle-detail.hpp" +#include "cryptonote_core/cryptonote_core.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.p2p" + +// ################################################################################################ +// local (TU local) headers +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + + std::string to_string(t_connection_type type) + { + if (type == e_connection_type_NET) + return std::string("NET"); + else if (type == e_connection_type_RPC) + return std::string("RPC"); + else if (type == e_connection_type_P2P) + return std::string("P2P"); + + return std::string("UNKNOWN"); + } + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + int m_peer_number; // e.g. for debug/stats +}; + + +} // namespace +} // namespace + +// ################################################################################################ +// The implementation part +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// connection_basic_pimpl +// ================================================================================================ + +connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { } + +// ================================================================================================ +// connection_basic +// ================================================================================================ + +// static variables: +int connection_basic_pimpl::m_default_tos; + +// methods: +connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number) + : + mI( new connection_basic_pimpl("peer") ), + strand_(io_service), + socket_(io_service), + m_want_close_connection(false), + m_was_shutdown(false), + m_ref_sock_count(ref_sock_count) +{ + ++ref_sock_count; // increase the global counter + mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number + + std::string remote_addr_str = "?"; + try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ; + + _note("Spawned connection p2p#"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count); + //boost::filesystem::create_directories("log/dr-monero/net/"); +} + +connection_basic::~connection_basic() noexcept(false) { + std::string remote_addr_str = "?"; + m_ref_sock_count--; + try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ; + _note("Destructing connection p2p#"<<mI->m_peer_number << " to " << remote_addr_str); +} + +void connection_basic::set_rate_up_limit(uint64_t limit) { + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().set_target_speed(limit); + } + save_limit_to_file(limit); +} + +void connection_basic::set_rate_down_limit(uint64_t limit) { + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().set_target_speed(limit); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().set_target_speed(limit); + } + save_limit_to_file(limit); +} + +uint64_t connection_basic::get_rate_up_limit() { + uint64_t limit; + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + limit = network_throttle_manager::get_global_throttle_out().get_target_speed(); + } + return limit; +} + +uint64_t connection_basic::get_rate_down_limit() { + uint64_t limit; + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + limit = network_throttle_manager::get_global_throttle_in().get_target_speed(); + } + return limit; +} + +void connection_basic::save_limit_to_file(int limit) { +} + +void connection_basic::set_tos_flag(int tos) { + connection_basic_pimpl::m_default_tos = tos; +} + +int connection_basic::get_tos_flag() { + return connection_basic_pimpl::m_default_tos; +} + +void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q_len) { + double delay=0; // will be calculated + do + { // rate limiting + if (m_was_shutdown) { + _dbg2("m_was_shutdown - so abort sleep"); + return; + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); + } + + delay *= 0.50; + if (delay > 0) { + long int ms = (long int)(delay * 1000); + MTRACE("Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // debug sleep + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); + } + } while(delay > 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_exact( packet_size ); // increase counter - global + } + +} +void connection_basic::set_start_time() { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds(); +} + +void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) { + // No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write + MTRACE("handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)"); + set_start_time(); +} + +void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) { + // No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write + MTRACE("handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)"); + + set_start_time(); +} + +void connection_basic::logger_handle_net_read(size_t size) { // network data read +} + +void connection_basic::logger_handle_net_write(size_t size) { +} + +double connection_basic::get_sleep_time(size_t cb) { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_out); + auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb); + return t; +} + +void connection_basic::set_save_graph(bool save_graph) { +} + + +} // namespace +} // namespace + diff --git a/contrib/epee/src/http_auth.cpp b/contrib/epee/src/http_auth.cpp index 30e562700..f06f05528 100644 --- a/contrib/epee/src/http_auth.cpp +++ b/contrib/epee/src/http_auth.cpp @@ -125,6 +125,14 @@ namespace { (*this)(boost::string_ref(arg)); } + void operator()(const epee::wipeable_string& arg) const + { + md5::MD5Update( + std::addressof(ctx), + reinterpret_cast<const std::uint8_t*>(arg.data()), + arg.size() + ); + } md5::MD5_CTX& ctx; }; diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index 74b7dd2f4..5b9472006 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -28,7 +28,11 @@ #ifndef _MLOG_H_ #define _MLOG_H_ +#include <time.h> #include <atomic> +#include <boost/filesystem.hpp> +#include <boost/algorithm/string.hpp> +#include "string_tools.h" #include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -55,6 +59,7 @@ static std::string generate_log_filename(const char *base) strcpy(tmp, "unknown"); else strftime(tmp, sizeof(tmp), "%Y-%m-%d-%H-%M-%S", &tm); + tmp[sizeof(tmp) - 1] = 0; filename += "-"; filename += tmp; return filename; diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index 22afcf819..2f4015e81 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -1,5 +1,6 @@ #include "net/net_utils_base.h" +#include "string_tools.h" #include <cstring> #include <typeindex> @@ -56,5 +57,34 @@ namespace epee { namespace net_utils if (typeid(*self_) != typeid(*other_self)) return false; return self_->is_same_host(*other_self); } + + bool create_network_address(network_address &address, const std::string &string, uint16_t default_port) + { + uint32_t ip; + uint16_t port; + if (epee::string_tools::parse_peer_from_string(ip, port, string)) + { + if (default_port && !port) + port = default_port; + address = ipv4_network_address{ip, port}; + return true; + } + return false; + } + + std::string print_connection_context(const connection_context_base& ctx) + { + std::stringstream ss; + ss << ctx.m_remote_address.str() << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT"); + return ss.str(); + } + + std::string print_connection_context_short(const connection_context_base& ctx) + { + std::stringstream ss; + ss << ctx.m_remote_address.str() << (ctx.m_is_income ? " INC":" OUT"); + return ss.str(); + } + }} diff --git a/contrib/epee/src/network_throttle-detail.cpp b/contrib/epee/src/network_throttle-detail.cpp new file mode 100644 index 000000000..ed6bc07ed --- /dev/null +++ b/contrib/epee/src/network_throttle-detail.cpp @@ -0,0 +1,371 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for throttle details */ + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "net/net_utils_base.h" +#include "misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + + + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> +#include "net/abstract_tcp_server2.h" + +// TODO: +#include "net/network_throttle-detail.hpp" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.throttle" + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + void _packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) could be used for different kinds of sleep e.g. direct/queue write +}; + + +} // namespace +} // namespace + + + + + + +// ################################################################################################ +// ################################################################################################ +// The implementation part +// ################################################################################################ +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle +// ================================================================================================ + +network_throttle::~network_throttle() { } + +network_throttle::packet_info::packet_info() + : m_size(0) +{ +} + +network_throttle::network_throttle(const std::string &nameshort, const std::string &name, int window_size) + : m_window_size( (window_size==-1) ? 10 : window_size ), + m_history( m_window_size ), m_nameshort(nameshort) +{ + set_name(name); + m_network_add_cost = 128; + m_network_minimal_segment = 256; + m_network_max_segment = 1024*1024; + m_any_packet_yet = false; + m_slot_size = 1.0; // hard coded in few places + m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle +} + +void network_throttle::set_name(const std::string &name) +{ + m_name = name; +} + +void network_throttle::set_target_speed( network_speed_kbps target ) +{ + m_target_speed = target * 1024; + MINFO("Setting LIMIT: " << target << " kbps"); +} + +network_speed_kbps network_throttle::get_target_speed() +{ + return m_target_speed / 1024; +} + +void network_throttle::tick() +{ + double time_now = get_time_seconds(); + if (!m_any_packet_yet) m_start_time = time_now; // starting now + + network_time_seconds current_sample_time_slot = time_to_slot( time_now ); // T=13.7 --> 13 (for 1-second smallwindow) + network_time_seconds last_sample_time_slot = time_to_slot( m_last_sample_time ); + + // moving to next position, and filling gaps + // !! during this loop the m_last_sample_time and last_sample_time_slot mean the variable moved in +1 + // TODO optimize when moving few slots at once + while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot)) + { + _dbg3("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); + // rotate buffer + for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1]; + m_history[0] = packet_info(); + if (! m_any_packet_yet) + { + m_last_sample_time = time_now; + } + m_last_sample_time += 1; last_sample_time_slot = time_to_slot( m_last_sample_time ); // increase and recalculate time, time slot + m_any_packet_yet=true; + } + m_last_sample_time = time_now; // the real exact last time +} + +void network_throttle::handle_trafic_exact(size_t packet_size) +{ + _handle_trafic_exact(packet_size, packet_size); +} + +void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size) +{ + tick(); + + calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1); + calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5); + m_history[0].m_size += packet_size; + + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + + MTRACE("Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)" + << " Speed AVG=" << std::setw(4) << ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]" + << " " << std::setw(4) << ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]" + <<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec " + << " " << history_str + ); +} + +void network_throttle::handle_trafic_tcp(size_t packet_size) +{ + size_t all_size = packet_size + m_network_add_cost; + all_size = std::max( m_network_minimal_segment , all_size); + _handle_trafic_exact( all_size , packet_size ); +} + +network_time_seconds network_throttle::get_sleep_time_after_tick(size_t packet_size) { + tick(); + return get_sleep_time(packet_size); +} + +void network_throttle::logger_handle_net(const std::string &filename, double time, size_t size) { + static boost::mutex mutex; + + boost::lock_guard<boost::mutex> lock(mutex); + { + std::fstream file; + file.open(filename.c_str(), std::ios::app | std::ios::out ); + file.precision(6); + if(!file.is_open()) + _warn("Can't open file " << filename); + file << static_cast<int>(time) << " " << static_cast<double>(size/1024) << "\n"; + file.close(); + } +} + +// fine tune this to decide about sending speed: +network_time_seconds network_throttle::get_sleep_time(size_t packet_size) const +{ + double D2=0; + calculate_times_struct cts = { 0, 0, 0, 0}; + calculate_times(packet_size, cts, true, m_window_size); D2=cts.delay; + return D2; +} + +// MAIN LOGIC: +void network_throttle::calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const +{ + const double the_window_size = std::max( (double)m_window_size , + ((force_window>0) ? force_window : m_window_size) + ); + + if (!m_any_packet_yet) { + cts.window=0; cts.average=0; cts.delay=0; + cts.recomendetDataSize = m_network_minimal_segment; // should be overrided by caller anyway + return ; // no packet yet, I can not decide about sleep time + } + + network_time_seconds window_len = (the_window_size-1) * m_slot_size ; // -1 since current slot is not finished + window_len += (m_last_sample_time - time_to_slot(m_last_sample_time)); // add the time for current slot e.g. 13.7-13 = 0.7 + + auto time_passed = get_time_seconds() - m_start_time; + cts.window = std::max( std::min( window_len , time_passed ) , m_slot_size ) ; // window length resulting from size of history but limited by how long ago history was started, + // also at least slot size (e.g. 1 second) to not be ridiculous + // window_len e.g. 5.7 because takes into account current slot time + + size_t Epast = 0; // summ of traffic till now + for (auto sample : m_history) Epast += sample.m_size; + + const size_t E = Epast; + const size_t Enow = Epast + packet_size ; // including the data we're about to send now + + const double M = m_target_speed; // max + const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed + const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet) + + cts.delay = (D1*0.80 + D2*0.20); // finall sleep depends on both with/without current packet + // update_overheat(); + cts.average = Epast/cts.window; // current avg. speed (for info) + + if (Epast <= 0) { + if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait + } + + double Wgood=-1; + { // how much data we recommend now to download + Wgood = the_window_size + 1; + cts.recomendetDataSize = M*cts.window - E; + } + + if (dbg) { + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + MTRACE((cts.delay > 0 ? "SLEEP" : "") + << "dbg " << m_name << ": " + << "speed is A=" << std::setw(8) <<cts.average<<" vs " + << "Max=" << std::setw(8) <<M<<" " + << " so sleep: " + << "D=" << std::setw(8) <<cts.delay<<" sec " + << "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") " + << "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " " + << "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " " + << "History: " << std::setw(8) << history_str << " " + << "m_last_sample_time=" << std::setw(8) << m_last_sample_time + ); + + } +} + +double network_throttle::get_time_seconds() const { + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif + auto time_from_epoh = point.time_since_epoch(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + return ms_f / 1000.; +} + +size_t network_throttle::get_recommended_size_of_planned_transport_window(double force_window) const { + calculate_times_struct cts = { 0, 0, 0, 0}; + network_throttle::calculate_times(0, cts, true, force_window); + cts.recomendetDataSize += m_network_add_cost; + if (cts.recomendetDataSize<0) cts.recomendetDataSize=0; + if (cts.recomendetDataSize>m_network_max_segment) cts.recomendetDataSize=m_network_max_segment; + size_t RI = (long int)cts.recomendetDataSize; + return RI; +} + +size_t network_throttle::get_recommended_size_of_planned_transport() const { + size_t R1=0,R2=0,R3=0; + R1 = get_recommended_size_of_planned_transport_window( -1 ); + R2 = get_recommended_size_of_planned_transport_window(m_window_size / 2); + R3 = get_recommended_size_of_planned_transport_window( 5 ); + auto RM = std::min(R1, std::min(R2,R3)); + + const double a1=20, a2=10, a3=10, am=10; // weight of the various windows in decisssion // TODO 70 => 20 + return (R1*a1 + R2*a2 + R3*a3 + RM*am) / (a1+a2+a3+am); +} + +double network_throttle::get_current_speed() const { + unsigned int bytes_transferred = 0; + if (m_history.size() == 0 || m_slot_size == 0) + return 0; + + auto it = m_history.begin(); + while (it < m_history.end() - 1) + { + bytes_transferred += it->m_size; + it ++; + } + + return bytes_transferred / ((m_history.size() - 1) * m_slot_size); +} + +} // namespace +} // namespace + diff --git a/contrib/epee/src/network_throttle.cpp b/contrib/epee/src/network_throttle.cpp new file mode 100644 index 000000000..dd1640a2e --- /dev/null +++ b/contrib/epee/src/network_throttle.cpp @@ -0,0 +1,110 @@ +/** +@file +@author rfree (current maintainer in monero.cc project) +@brief interface for throttling of connection (count and rate-limit speed etc) +@details <PRE> + +Throttling work by: +1) taking note of all traffic (hooks added e.g. to connection class) and measuring speed +2) depending on that information we sleep before sending out data (or send smaller portions of data) +3) depending on the information we can also sleep before sending requests or ask for smaller sets of data to download + +</PRE> + +@image html images/net/rate1-down-1k.png +@image html images/net/rate1-down-full.png +@image html images/net/rate1-up-10k.png +@image html images/net/rate1-up-full.png +@image html images/net/rate2-down-100k.png +@image html images/net/rate2-down-10k.png +@image html images/net/rate2-down-50k.png +@image html images/net/rate2-down-full.png +@image html images/net/rate2-up-100k.png +@image html images/net/rate2-up-10k.png +@image html images/net/rate3-up-10k.png + + +*/ + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/network_throttle-detail.hpp" + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle_manager +// ================================================================================================ + +// ================================================================================================ +// static: +boost::mutex network_throttle_manager::m_lock_get_global_throttle_in; +boost::mutex network_throttle_manager::m_lock_get_global_throttle_inreq; +boost::mutex network_throttle_manager::m_lock_get_global_throttle_out; + +// ================================================================================================ +// methods: +i_network_throttle & network_throttle_manager::get_global_throttle_in() { + static network_throttle obj_get_global_throttle_in("in/all","<<< global-IN",10); + return obj_get_global_throttle_in; +} + + + +i_network_throttle & network_throttle_manager::get_global_throttle_inreq() { + static network_throttle obj_get_global_throttle_inreq("inreq/all", "<== global-IN-REQ",10); + return obj_get_global_throttle_inreq; +} + + +i_network_throttle & network_throttle_manager::get_global_throttle_out() { + static network_throttle obj_get_global_throttle_out("out/all", ">>> global-OUT",10); + return obj_get_global_throttle_out; +} + + + + +network_throttle_bw::network_throttle_bw(const std::string &name1) + : m_in("in/"+name1, name1+"-DOWNLOAD"), m_inreq("inreq/"+name1, name1+"-DOWNLOAD-REQUESTS"), m_out("out/"+name1, name1+"-UPLOAD") +{ } + + + + +} // namespace +} // namespace + + + + + diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index 291bba94c..00c2ddd62 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -3,6 +3,7 @@ #include <readline/history.h> #include <sys/select.h> #include <unistd.h> +#include <iostream> #include <boost/thread.hpp> #include <boost/algorithm/string.hpp> diff --git a/contrib/epee/src/string_tools.cpp b/contrib/epee/src/string_tools.cpp index d04b16b75..fd0254016 100644 --- a/contrib/epee/src/string_tools.cpp +++ b/contrib/epee/src/string_tools.cpp @@ -26,6 +26,8 @@ #include "string_tools.h" +#include <ctype.h> + #ifdef _WIN32 # include <winsock2.h> #else @@ -56,6 +58,16 @@ namespace string_tools return true; } + //---------------------------------------------------------------------------- + bool validate_hex(uint64_t length, const std::string& str) + { + if (str.size() != length) + return false; + for (char c: str) + if (!isxdigit(c)) + return false; + return true; + } } } diff --git a/contrib/epee/src/wipeable_string.cpp b/contrib/epee/src/wipeable_string.cpp new file mode 100644 index 000000000..894c47bbd --- /dev/null +++ b/contrib/epee/src/wipeable_string.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string.h> +#include "misc_log_ex.h" +#include "wipeable_string.h" + +namespace epee +{ + +void *(*wipeable_string::wipefunc)(void*, size_t) = NULL; + +wipeable_string::wipeable_string(const wipeable_string &other): + buffer(other.buffer) +{ +} + +wipeable_string::wipeable_string(wipeable_string &&other) +{ + if (&other == this) + return; + buffer = std::move(other.buffer); +} + +wipeable_string::wipeable_string(const std::string &other) +{ + grow(other.size()); + memcpy(buffer.data(), other.c_str(), size()); +} + +wipeable_string::wipeable_string(std::string &&other) +{ + CHECK_AND_ASSERT_THROW_MES(wipefunc, "wipefunc is not set"); + grow(other.size()); + memcpy(buffer.data(), other.c_str(), size()); + if (!other.empty()) + { + wipefunc(&other[0], other.size()); // we're kinda left with this again aren't we + other = std::string(); + } +} + +wipeable_string::wipeable_string(const char *s) +{ + grow(strlen(s)); + memcpy(buffer.data(), s, size()); +} + +wipeable_string::~wipeable_string() +{ + wipe(); +} + +void wipeable_string::wipe() +{ + CHECK_AND_ASSERT_THROW_MES(wipefunc, "wipefunc is not set"); + wipefunc(buffer.data(), buffer.size() * sizeof(char)); +} + +void wipeable_string::grow(size_t sz, size_t reserved) +{ + CHECK_AND_ASSERT_THROW_MES(wipefunc, "wipefunc is not set"); + if (reserved < sz) + reserved = sz; + if (reserved <= buffer.capacity()) + { + if (sz < buffer.size()) + wipefunc(buffer.data() + sz, buffer.size() - sz); + buffer.resize(sz); + return; + } + size_t old_sz = buffer.size(); + std::unique_ptr<char[]> tmp{new char[old_sz]}; + memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); + wipefunc(buffer.data(), old_sz * sizeof(char)); + buffer.reserve(reserved); + buffer.resize(sz); + memcpy(buffer.data(), tmp.get(), sz * sizeof(char)); + wipefunc(tmp.get(), old_sz * sizeof(char)); +} + +void wipeable_string::push_back(char c) +{ + grow(size() + 1); + buffer.back() = c; +} + +void wipeable_string::pop_back() +{ + resize(size() - 1); +} + +void wipeable_string::resize(size_t sz) +{ + grow(sz); +} + +void wipeable_string::reserve(size_t sz) +{ + grow(size(), sz); +} + +void wipeable_string::clear() +{ + resize(0); +} + +wipeable_string &wipeable_string::operator=(wipeable_string &&other) +{ + if (&other != this) + buffer = std::move(other.buffer); + return *this; +} + +wipeable_string &wipeable_string::operator=(const wipeable_string &other) +{ + if (&other != this) + buffer = other.buffer; + return *this; +} + +} diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index 35b74f7e4..f1c4ff202 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -10,12 +10,20 @@ fi type="$1" if test -z "$type" then - echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction" + echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin" exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin"; exit 1 ;; esac -afl-fuzz -i tests/data/fuzz/$type -m 150 -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests +if test -d "fuzz-out/$type" +then + dir="-" +else + dir="tests/data/fuzz/$type" +fi + +mkdir -p fuzz-out +afl-fuzz -i "$dir" -m none -t 250 -o fuzz-out/$type build/fuzz/tests/fuzz/${type}_fuzz_tests @@ |