diff options
Diffstat (limited to 'contrib/epee/include')
26 files changed, 594 insertions, 92 deletions
diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 6d369d4d8..4ea3fa54b 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -27,6 +27,7 @@ #pragma once #include "misc_log_ex.h" +#include "string_tools.h" #include <atomic> #include <condition_variable> #include <functional> @@ -37,6 +38,8 @@ #include <stdio.h> #endif #include <boost/thread.hpp> +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> #ifdef HAVE_READLINE #include "readline_buffer.h" diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index c387743a6..f037b4dd5 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -29,7 +29,8 @@ #define _FILE_IO_UTILS_H_ #include <iostream> -#include <boost/filesystem.hpp> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> namespace epee { @@ -63,7 +64,7 @@ namespace file_io_utils } inline - bool get_file_time(const std::string& path_to_file, OUT time_t& ft) + bool get_file_time(const std::string& path_to_file, time_t& ft) { boost::system::error_code ec; ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); diff --git a/contrib/epee/include/math_helper.h b/contrib/epee/include/math_helper.h index 90398acbb..ef839f609 100644 --- a/contrib/epee/include/math_helper.h +++ b/contrib/epee/include/math_helper.h @@ -37,6 +37,7 @@ #include <boost/uuid/random_generator.hpp> #include "misc_os_dependent.h" +#include "syncobj.h" namespace epee { diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 7ac07d112..35ec0950b 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -28,26 +28,7 @@ #ifndef _MISC_LOG_EX_H_ #define _MISC_LOG_EX_H_ -#include "static_initializer.h" -#include "string_tools.h" -#include "time_helper.h" -#include "misc_os_dependent.h" - -#include "syncobj.h" - -#include <iostream> -#include <sstream> -#include <iomanip> -#include <fstream> -#include <algorithm> -#include <list> -#include <map> #include <string> -#include <time.h> -#include <boost/cstdint.hpp> -#include <boost/thread.hpp> -#include <boost/filesystem.hpp> -#include <boost/algorithm/string.hpp> #include "easylogging++.h" @@ -169,7 +150,7 @@ namespace debug #define ASSERT_MES_AND_THROW(message) {LOG_ERROR(message); std::stringstream ss; ss << message; throw std::runtime_error(ss.str());} -#define CHECK_AND_ASSERT_THROW_MES(expr, message) {if(!(expr)) ASSERT_MES_AND_THROW(message);} +#define CHECK_AND_ASSERT_THROW_MES(expr, message) do {if(!(expr)) ASSERT_MES_AND_THROW(message);} while(0) #ifndef CHECK_AND_ASSERT diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 69ded09e5..99690b301 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -23,6 +23,10 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // +#ifdef _WIN32 +#include <Winsock2.h> +#endif + #ifdef WIN32 #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -42,6 +46,9 @@ #include <mach/mach.h> #endif +#include <iostream> +#include <boost/lexical_cast.hpp> + #pragma once namespace epee { diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 33fec8ec5..c0401c8b0 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -54,8 +54,8 @@ #include <boost/thread/thread.hpp> #include "net_utils_base.h" #include "syncobj.h" -#include "../../../../src/p2p/connection_basic.hpp" -#include "../../../../src/p2p/network_throttle-detail.hpp" +#include "connection_basic.hpp" +#include "network_throttle-detail.hpp" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 00d03567c..04d884af2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -286,7 +286,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); - epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred * 1024); + epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred); } double delay=0; // will be calculated - how much we should sleep to obey speed limit etc @@ -297,7 +297,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { { //_scope_dbg1("CRITICAL_REGION_LOCAL"); CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); - delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); // decission from global throttle + delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); } delay *= 0.5; @@ -482,9 +482,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //some data should be wrote to stream //request complete - if (speed_limit_is_enabled()) { - sleep_before_packet(cb, 1, 1); - } + // No sleeping here; sleeping is done once and for all in "handle_write" m_send_que_lock.lock(); // *** critical *** epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_send_que_lock.unlock();}); @@ -607,6 +605,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } logger_handle_net_write(cb); + // The single sleeping that is needed for correctly handling "out" speed throttling if (speed_limit_is_enabled()) { sleep_before_packet(cb, 1, 1); } diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp new file mode 100644 index 000000000..16de469a7 --- /dev/null +++ b/contrib/epee/include/net/connection_basic.hpp @@ -0,0 +1,141 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// ! This file might contain variable names same as in template class connection<> +// ! from files contrib/epee/include/net/abstract_tcp_server2.* +// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;) +// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) +// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#ifndef INCLUDED_p2p_connection_basic_hpp +#define INCLUDED_p2p_connection_basic_hpp + + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "net/net_utils_base.h" +#include "syncobj.h" + +namespace epee +{ +namespace net_utils +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + /// Represents a single connection from a client. + +class connection_basic_pimpl; // PIMPL for this class + + enum t_connection_type { // type of the connection (of this server), e.g. so that we will know how to limit it + e_connection_type_NET = 0, // default (not used?) + e_connection_type_RPC = 1, // the rpc commands (probably not rate limited, not chunked, etc) + e_connection_type_P2P = 2 // to other p2p node (probably limited) + }; + + std::string to_string(t_connection_type type); + +class connection_basic { // not-templated base class for rapid developmet of some code parts + public: + std::unique_ptr< connection_basic_pimpl > mI; // my Implementation + + // moved here from orginal connecton<> - common member variables that do not depend on template in connection<> + volatile uint32_t m_want_close_connection; + std::atomic<bool> m_was_shutdown; + critical_section m_send_que_lock; + std::list<std::string> m_send_que; + volatile bool m_is_multithreaded; + double m_start_time; + /// Strand to ensure the connection's handlers are not called concurrently. + boost::asio::io_service::strand strand_; + /// Socket for the connection. + boost::asio::ip::tcp::socket socket_; + + std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/-- + public: + // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator + connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number); + + virtual ~connection_basic() noexcept(false); + + // various handlers to be called from connection class: + void do_send_handler_write(const void * ptr , size_t cb); + void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part + + void logger_handle_net_write(size_t size); // network data written + void logger_handle_net_read(size_t size); // network data read + + void set_start_time(); + + // config for rate limit + + static void set_rate_up_limit(uint64_t limit); + static void set_rate_down_limit(uint64_t limit); + static uint64_t get_rate_up_limit(); + static uint64_t get_rate_down_limit(); + + // config misc + static void set_tos_flag(int tos); // ToS / QoS flag + static int get_tos_flag(); + + // handlers and sleep + void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) + static void save_limit_to_file(int limit); ///< for dr-monero + static double get_sleep_time(size_t cb); + + static void set_save_graph(bool save_graph); +}; + +} // nameserver +} // nameserver + +#endif + + diff --git a/contrib/epee/include/net/http_auth.h b/contrib/epee/include/net/http_auth.h index bf368e6f4..841cebc17 100644 --- a/contrib/epee/include/net/http_auth.h +++ b/contrib/epee/include/net/http_auth.h @@ -33,7 +33,7 @@ #include <functional> #include <string> #include <utility> - +#include "wipeable_string.h" #include "http_base.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -48,12 +48,12 @@ namespace net_utils struct login { login() : username(), password() {} - login(std::string username_, std::string password_) + login(std::string username_, wipeable_string password_) : username(std::move(username_)), password(std::move(password_)) {} std::string username; - std::string password; + wipeable_string password; }; //! Implements RFC 2617 digest auth. Digests from RFC 7616 can be added. diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index d77c84c28..1a9d5d064 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -50,6 +50,7 @@ #include "http_auth.h" #include "to_nonconst_iterator.h" #include "net_parse_helpers.h" +#include "syncobj.h" //#include "shlwapi.h" diff --git a/contrib/epee/include/net/http_client_base.h b/contrib/epee/include/net/http_client_base.h index f5fb57d03..c3da28718 100644 --- a/contrib/epee/include/net/http_client_base.h +++ b/contrib/epee/include/net/http_client_base.h @@ -38,8 +38,8 @@ namespace epee virtual ~i_sub_handler(){} virtual bool update_in( std::string& piece_of_transfer)=0; - virtual void stop(std::string& OUT collect_remains)=0; - virtual bool update_and_stop(std::string& OUT collect_remains, bool& is_changed) + virtual void stop(std::string& collect_remains)=0; + virtual bool update_and_stop(std::string& collect_remains, bool& is_changed) { is_changed = true; bool res = this->update_in(collect_remains); @@ -66,7 +66,7 @@ namespace epee { return m_powner_filter->handle_target_data(piece_of_transfer); } - virtual void stop(std::string& OUT collect_remains) + virtual void stop(std::string& collect_remains) { } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 7ad6d198b..7b526e4e7 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -35,6 +35,8 @@ #include "levin_base.h" #include "misc_language.h" +#include "syncobj.h" +#include "misc_os_dependent.h" #include <random> #include <chrono> diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index ee0e13fc1..2c2efcd82 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -31,23 +31,16 @@ //#include <Winsock2.h> //#include <Ws2tcpip.h> -#include <boost/lexical_cast.hpp> -#include <iostream> -#include <istream> -#include <ostream> #include <string> #include <boost/version.hpp> #include <boost/asio.hpp> #include <boost/asio/ssl.hpp> #include <boost/asio/steady_timer.hpp> -#include <boost/preprocessor/selection/min.hpp> #include <boost/lambda/bind.hpp> #include <boost/lambda/lambda.hpp> #include <boost/interprocess/detail/atomic.hpp> #include "net/net_utils_base.h" #include "misc_language.h" -//#include "profile_tools.h" -#include "../string_tools.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 0e31ee86f..7615786be 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -29,13 +29,11 @@ #ifndef _NET_UTILS_BASE_H_ #define _NET_UTILS_BASE_H_ -#include <boost/asio/io_service.hpp> #include <boost/uuid/uuid.hpp> -#include <memory> +#include <boost/asio/io_service.hpp> #include <typeinfo> #include <type_traits> #include "serialization/keyvalue_serialization.h" -#include "string_tools.h" #include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -45,7 +43,6 @@ #define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) #endif - namespace epee { namespace net_utils @@ -166,15 +163,37 @@ namespace net_utils BEGIN_KV_SERIALIZE_MAP() uint8_t type = is_store ? this_ref.get_type_id() : 0; - epee::serialization::selector<is_store>::serialize(type, stg, hparent_section, "type"); + if (!epee::serialization::selector<is_store>::serialize(type, stg, hparent_section, "type")) + return false; switch (type) { case ipv4_network_address::ID: + { if (!is_store) + { const_cast<network_address&>(this_ref) = ipv4_network_address{0, 0}; - KV_SERIALIZE(template as_mutable<ipv4_network_address>()); + auto &addr = this_ref.template as_mutable<ipv4_network_address>(); + if (epee::serialization::selector<is_store>::serialize(addr, stg, hparent_section, "addr")) + MDEBUG("Found as addr: " << this_ref.str()); + else if (epee::serialization::selector<is_store>::serialize(addr, stg, hparent_section, "template as<ipv4_network_address>()")) + MDEBUG("Found as template as<ipv4_network_address>(): " << this_ref.str()); + else if (epee::serialization::selector<is_store>::serialize(addr, stg, hparent_section, "template as_mutable<ipv4_network_address>()")) + MDEBUG("Found as template as_mutable<ipv4_network_address>(): " << this_ref.str()); + else + { + MWARNING("Address not found"); + return false; + } + } + else + { + auto &addr = this_ref.template as_mutable<ipv4_network_address>(); + if (!epee::serialization::selector<is_store>::serialize(addr, stg, hparent_section, "addr")) + return false; + } break; - default: MERROR("Unsupported network address type: " << type); return false; + } + default: MERROR("Unsupported network address type: " << (unsigned)type); return false; } END_KV_SERIALIZE_MAP() }; @@ -192,19 +211,8 @@ namespace net_utils inline bool operator>=(const network_address& lhs, const network_address& rhs) { return !lhs.less(rhs); } - inline bool create_network_address(network_address &address, const std::string &string, uint16_t default_port = 0) - { - uint32_t ip; - uint16_t port; - if (epee::string_tools::parse_peer_from_string(ip, port, string)) - { - if (default_port && !port) - port = default_port; - address = ipv4_network_address{ip, port}; - return true; - } - return false; - } + bool create_network_address(network_address &address, const std::string &string, uint16_t default_port = 0); + /************************************************************************/ /* */ /************************************************************************/ @@ -287,21 +295,8 @@ namespace net_utils //some helpers - inline - std::string print_connection_context(const connection_context_base& ctx) - { - std::stringstream ss; - ss << ctx.m_remote_address.str() << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT"); - return ss.str(); - } - - inline - std::string print_connection_context_short(const connection_context_base& ctx) - { - std::stringstream ss; - ss << ctx.m_remote_address.str() << (ctx.m_is_income ? " INC":" OUT"); - return ss.str(); - } + std::string print_connection_context(const connection_context_base& ctx); + std::string print_connection_context_short(const connection_context_base& ctx); inline MAKE_LOGGABLE(connection_context_base, ct, os) { diff --git a/contrib/epee/include/net/network_throttle-detail.hpp b/contrib/epee/include/net/network_throttle-detail.hpp new file mode 100644 index 000000000..dba15a5ed --- /dev/null +++ b/contrib/epee/include/net/network_throttle-detail.hpp @@ -0,0 +1,125 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: throttle details, implementing rate limiting */ + + +#ifndef INCLUDED_throttle_detail_hpp +#define INCLUDED_throttle_detail_hpp + +#include "network_throttle.hpp" + +namespace epee +{ +namespace net_utils +{ + + +class network_throttle : public i_network_throttle { + private: + struct packet_info { + size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second) + packet_info(); + }; + + + network_speed_bps m_target_speed; + size_t m_network_add_cost; // estimated add cost of headers + size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to + size_t m_network_max_segment; // recommended max size of 1 TCP transmission + + const size_t m_window_size; // the number of samples to average over + network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot() + // TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements + + std::vector< packet_info > m_history; // the history of bw usage + network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer + network_time_seconds m_start_time; // when we were created + bool m_any_packet_yet; // did we yet got any packet to count + + std::string m_name; // my name for debug and logs + std::string m_nameshort; // my name for debug and logs (used in log file name) + + // each sample is now 1 second + public: + network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1); + virtual ~network_throttle(); + virtual void set_name(const std::string &name); + virtual void set_target_speed( network_speed_kbps target ); + virtual network_speed_kbps get_target_speed(); + + // add information about events: + virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc + + virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc) + + virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic + + // time calculations: + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info) + + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size + virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO + + virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported + virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame + virtual double get_current_speed() const; + + private: + virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 + virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size); + virtual void logger_handle_net(const std::string &filename, double time, size_t size); +}; + +/*** + * The complete set of traffic throttle for one typical connection +*/ +struct network_throttle_bw { + public: + network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually) + network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually) + network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually) + + public: + network_throttle_bw(const std::string &name1); +}; + + + +} // namespace net_utils +} // namespace epee + + +#endif + + diff --git a/contrib/epee/include/net/network_throttle.hpp b/contrib/epee/include/net/network_throttle.hpp new file mode 100644 index 000000000..464b34726 --- /dev/null +++ b/contrib/epee/include/net/network_throttle.hpp @@ -0,0 +1,176 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief interface for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014-2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: throttle basic interface */ +/* rfree: also includes the manager for singeton/global such objects */ + + +#ifndef INCLUDED_network_throttle_hpp +#define INCLUDED_network_throttle_hpp + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include "syncobj.h" + +#include "net/net_utils_base.h" +#include "misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + +#include <memory> +#include <mutex> +#include <fstream> + +namespace epee +{ +namespace net_utils +{ + +// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono? +typedef double network_speed_kbps; // externally, for parameters and return values, all defined in kilobytes per second +typedef double network_speed_bps; // throttle-internally, bytes per second +typedef double network_time_seconds; +typedef double network_MB; + +class i_network_throttle; + +/*** +@brief All information about given throttle - speed calculations +*/ +struct calculate_times_struct { + double average; + double window; + double delay; + double recomendetDataSize; +}; +typedef calculate_times_struct calculate_times_struct; + + +namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working + +/*** +@brief Access to simple throttles, with singlton to access global network limits +*/ +class network_throttle_manager { + // provides global (singleton) in/inreq/out throttle access + + // [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/ + // [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests) + + //protected: + public: // XXX + + static boost::mutex m_lock_get_global_throttle_in; + static boost::mutex m_lock_get_global_throttle_inreq; + static boost::mutex m_lock_get_global_throttle_out; + + friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic_pimpl; // ditto + + static int xxx; + + public: + static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in + static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously + static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously +}; + + + +/*** +@brief interface for the throttle, see the derivated class +*/ +class i_network_throttle { + public: + virtual void set_name(const std::string &name)=0; + virtual void set_target_speed( network_speed_kbps target )=0; + virtual network_speed_kbps get_target_speed()=0; + + virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc + virtual void tick() =0; // poke and update timers/history + + // time calculations: + + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate: + // Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size + + // Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now + + virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer + + virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future + + virtual double get_time_seconds() const =0; // a timer + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + + +}; + + +// ... more in the -advanced.h file + + +} // namespace net_utils +} // namespace epee + + +#endif + + + diff --git a/contrib/epee/include/profile_tools.h b/contrib/epee/include/profile_tools.h index f285fe48b..a0b5f77f4 100644 --- a/contrib/epee/include/profile_tools.h +++ b/contrib/epee/include/profile_tools.h @@ -28,6 +28,8 @@ #ifndef _PROFILE_TOOLS_H_ #define _PROFILE_TOOLS_H_ +#include "misc_os_dependent.h" + namespace epee { diff --git a/contrib/epee/include/readline_buffer.h b/contrib/epee/include/readline_buffer.h index cda7e34f9..87c8826cb 100644 --- a/contrib/epee/include/readline_buffer.h +++ b/contrib/epee/include/readline_buffer.h @@ -2,9 +2,7 @@ #include <streambuf> #include <sstream> -#include <iostream> #include <vector> -#include <algorithm> namespace rdln { diff --git a/contrib/epee/include/reg_exp_definer.h b/contrib/epee/include/reg_exp_definer.h index e2bed5c3f..eb11c9e10 100644 --- a/contrib/epee/include/reg_exp_definer.h +++ b/contrib/epee/include/reg_exp_definer.h @@ -29,7 +29,7 @@ #define _REG_EXP_DEFINER_H_ #include <boost/interprocess/detail/atomic.hpp> - +#include "syncobj.h" namespace epee { diff --git a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h index a94ecacc5..7087136cc 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h +++ b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h @@ -26,6 +26,13 @@ #pragma once +#include <set> +#include <list> +#include <vector> +#include <deque> +#include <boost/mpl/vector.hpp> +#include <boost/mpl/contains_fwd.hpp> + namespace epee { namespace serialization @@ -73,7 +80,7 @@ namespace epee template<class serializible_type, class t_storage> static bool unserialize_t_obj(serializible_type& obj, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname) { - typename t_storage::hsection hchild_section = stg.open_section(pname, hparent_section, true); + typename t_storage::hsection hchild_section = stg.open_section(pname, hparent_section, false); if(!hchild_section) return false; return obj._load(stg, hchild_section); } @@ -90,7 +97,7 @@ namespace epee static bool unserialize_t_obj(enableable<serializible_type>& obj, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname) { obj.enabled = false; - typename t_storage::hsection hchild_section = stg.open_section(pname, hparent_section, true); + typename t_storage::hsection hchild_section = stg.open_section(pname, hparent_section, false); if(!hchild_section) return false; obj.enabled = true; return obj.v._load(stg, hchild_section); diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index ea4ba63dd..5660c09d1 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -108,7 +108,7 @@ namespace epee template<typename T> constexpr bool has_padding() noexcept { - return !std::is_pod<T>() || alignof(T) != 1; + return !std::is_pod<T>::value || alignof(T) != 1; } //! \return Cast data from `src` as `span<const std::uint8_t>`. diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 04b57376c..727f36552 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -25,6 +25,8 @@ // #pragma once +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "parserse_base_utils.h" #include "file_io_utils.h" diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 38e9f9098..9501bbc2a 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -28,6 +28,7 @@ #pragma once +#include "pragma_comp_defs.h" #include "misc_language.h" #include "portable_storage_base.h" diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index f4a16cfae..52aa09eba 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -28,7 +28,7 @@ #pragma once -#include <regex> +#include <boost/regex.hpp> #include "misc_language.h" #include "portable_storage_base.h" @@ -146,7 +146,7 @@ POP_WARNINGS if(std::all_of(from.begin(), from.end(), ::isdigit)) to = boost::lexical_cast<uint64_t>(from); // MyMonero ISO 8061 timestamp (2017-05-06T16:27:06Z) - else if (std::regex_match (from, std::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ"))) + else if (boost::regex_match (from, boost::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ"))) { // Convert to unix timestamp std::tm tm = {}; diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 25639263c..307323aa1 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -35,15 +35,15 @@ # include <windows.h> #endif +#include <string.h> #include <locale> #include <cstdlib> #include <string> #include <type_traits> -#include <regex> #include <boost/uuid/uuid.hpp> #include <boost/uuid/uuid_io.hpp> #include <boost/lexical_cast.hpp> -#include <boost/algorithm/string.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "hex.h" #include "span.h" #include "warnings.h" @@ -350,11 +350,8 @@ POP_WARNINGS s = *(t_pod_type*)bin_buff.data(); return true; } - //---------------------------------------------------------------------------- - inline bool validate_hex(uint64_t length, const std::string& str) - { - return std::regex_match(str, std::regex("'^[0-9abcdefABCDEF]+$'")) && str.size() == length; - } + //---------------------------------------------------------------------------- + bool validate_hex(uint64_t length, const std::string& str); //---------------------------------------------------------------------------- inline std::string get_extension(const std::string& str) { diff --git a/contrib/epee/include/wipeable_string.h b/contrib/epee/include/wipeable_string.h new file mode 100644 index 000000000..66d3e8e2b --- /dev/null +++ b/contrib/epee/include/wipeable_string.h @@ -0,0 +1,70 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stddef.h> +#include <vector> +#include <string> + +namespace epee +{ + class wipeable_string + { + public: + wipeable_string() {} + wipeable_string(const wipeable_string &other); + wipeable_string(wipeable_string &&other); + wipeable_string(const std::string &other); + wipeable_string(std::string &&other); + wipeable_string(const char *s); + ~wipeable_string(); + void wipe(); + void push_back(char c); + void pop_back(); + const char *data() const noexcept { return buffer.data(); } + size_t size() const noexcept { return buffer.size(); } + bool empty() const noexcept { return buffer.empty(); } + void resize(size_t sz); + void reserve(size_t sz); + void clear(); + bool operator==(const wipeable_string &other) const noexcept { return buffer == other.buffer; } + bool operator!=(const wipeable_string &other) const noexcept { return buffer != other.buffer; } + wipeable_string &operator=(wipeable_string &&other); + wipeable_string &operator=(const wipeable_string &other); + + static void set_wipe(void *(*f)(void*, size_t)) { wipefunc = f; } + + private: + void grow(size_t sz, size_t reserved = 0); + + private: + std::vector<char> buffer; + static void *(*wipefunc)(void*, size_t); + }; +} |