From eabb519605cab00dbaa5a1868d229f09c74570a6 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Mon, 5 Jan 2015 20:30:17 +0100 Subject: 2014 network limit 1.0a +utils +toc -doc -drmonero commands and options for network limiting works very well e.g. for 50 KiB/sec up and down ToS (QoS) flag peer number limit TODO some spikes in ingress/download TODO problems when other up and down limit added "otshell utils" - simple logging (with colors, text files channels) --- .../cryptonote_protocol_handler-base.cpp | 266 +++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp (limited to 'src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp') diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp new file mode 100644 index 000000000..6b25cb681 --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -0,0 +1,266 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include +#include +#include + + +#include +#include + +#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h" +#include "../../src/p2p/network_throttle.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +class cryptonote_protocol_handler_base_pimpl { // placeholer if needed + public: + +}; + +} // namespace + +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa + const double size_min = 500; // XXX 500 + const int history_len = 20; // how many blocks to average over + + double avg=0; + try { + avg = get_avg_block_size(history_len); + } catch (...) { } + avg = std::max( size_min , avg); + return avg; +} + +cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { +} + +cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { +} + +void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) { + using namespace epee::net_utils; + size_t est_req_size=0; // how much data are we now requesting (to be soon send to us) + + const auto count_limit_default = count_limit; + + bool allowed_now = false; // are we now allowed to request or are we limited still + // long int size_limit; + + while (!allowed_now) { + /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit + _fact("ABORT sleep (before sending requeset) due to stopping"); + break; + }*/ + + //LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0); + //{ + long int size_limit1=0, size_limit2=0; + //LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0); + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().tick(); + size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport(); + } + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().tick(); + size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport(); + } + + long int one_block_estimated_size = estimate_one_block_size(); + long int limit_small = std::min( size_limit1 , size_limit2 ); + long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3; + if (limit_small <= 0) size_limit = 0; + const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread + const double knob = 1.000; + size_limit /= (estimated_peers / estimated_peers) * knob; + _note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<& ids) { + using namespace epee::net_utils; + LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0); + _note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + // TODO +} + +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_mark(""); + using namespace epee::net_utils; + double delay=0; // will be calculated + _dbg1("Packet size: " << packet_size); + do + { // rate limiting + //XXX + /*if (::cryptonote::core::get_is_stopping()) { + _dbg1("We are stopping - so abort sleep"); + return; + }*/ + /*if (m_was_shutdown) { + _dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep"); + return; + }*/ + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + + delay *= 0.50; + //delay = 0; // XXX + if (delay > 0) { + //delay += rand2*0.1; + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global + //epee::critical_region_t guard(m_throttle_global_lock); // *** critical *** + //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + } +} + +} // namespace + + -- cgit v1.2.3 From 5ce4256e3d6ff2e1595750e3875865089e20a03b Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Thu, 12 Feb 2015 20:59:39 +0100 Subject: 2014 network limit 1.1 +utils +toc -doc -drmonero Update of the PR with network limits works very well for all speeds (but remember that low download speed can stop upload because we then slow down downloading of blockchain requests too) more debug options fixed pedantic warnings in our code should work again on Mac OS X and FreeBSD fixed warning about size_t tested on Debian, Ubuntu, Windows(testing now) TCP options and ToS (QoS) flag FIXED peer number limit FIXED some spikes in ingress/download FIXED problems when other up and down limit --- .../cryptonote_protocol_handler-base.cpp | 94 +--------------------- 1 file changed, 2 insertions(+), 92 deletions(-) (limited to 'src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp') diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index 6b25cb681..b5a5ceea9 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -104,11 +104,11 @@ namespace cryptonote { double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa const double size_min = 500; // XXX 500 - const int history_len = 20; // how many blocks to average over + //const int history_len = 20; // how many blocks to average over double avg=0; try { - avg = get_avg_block_size(history_len); + avg = get_avg_block_size(/*history_len*/); } catch (...) { } avg = std::max( size_min , avg); return avg; @@ -120,96 +120,6 @@ cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { } -void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) { - using namespace epee::net_utils; - size_t est_req_size=0; // how much data are we now requesting (to be soon send to us) - - const auto count_limit_default = count_limit; - - bool allowed_now = false; // are we now allowed to request or are we limited still - // long int size_limit; - - while (!allowed_now) { - /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit - _fact("ABORT sleep (before sending requeset) due to stopping"); - break; - }*/ - - //LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0); - //{ - long int size_limit1=0, size_limit2=0; - //LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0); - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); - network_throttle_manager::get_global_throttle_in().tick(); - size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport(); - } - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); - network_throttle_manager::get_global_throttle_inreq().tick(); - size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport(); - } - - long int one_block_estimated_size = estimate_one_block_size(); - long int limit_small = std::min( size_limit1 , size_limit2 ); - long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3; - if (limit_small <= 0) size_limit = 0; - const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread - const double knob = 1.000; - size_limit /= (estimated_peers / estimated_peers) * knob; - _note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<& ids) { using namespace epee::net_utils; LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); -- cgit v1.2.3 From ae2a50659f7dc74a5446a0dc3a5f8f78563b9e1f Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Fri, 20 Feb 2015 22:28:03 +0100 Subject: 2014 network limit 1.2 +utils +toc -doc -drmonero new update of the pr with network limits more debug options: discarding downloaded blocks all or after given height. trying to trigger the locking errors. debug levels polished/tuned to sane values. debug/logging improved. warning: this pr should be correct code, but it could make an existing (in master version) locking error appear more often. it's a race on the list (map) of peers, e.g. between closing/deleting them versus working on them in net-limit sleep in sending chunk. the bug is not in this code/this pr, but in the master version. the locking problem of master will be fixed in other pr. problem is ub, and in practice is seems to usually cause program abort (tested on debian stable with updated gcc). see --help for option to add sleep to trigger the error faster. --- src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp') diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index b5a5ceea9..614ee8fab 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -128,7 +128,7 @@ void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list< // TODO } -void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_mark(""); +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_dbg1(""); using namespace epee::net_utils; double delay=0; // will be calculated _dbg1("Packet size: " << packet_size); -- cgit v1.2.3