From eabb519605cab00dbaa5a1868d229f09c74570a6 Mon Sep 17 00:00:00 2001 From: rfree2monero Date: Mon, 5 Jan 2015 20:30:17 +0100 Subject: 2014 network limit 1.0a +utils +toc -doc -drmonero commands and options for network limiting works very well e.g. for 50 KiB/sec up and down ToS (QoS) flag peer number limit TODO some spikes in ingress/download TODO problems when other up and down limit added "otshell utils" - simple logging (with colors, text files channels) --- src/cryptonote_protocol/CMakeLists.txt | 46 ++++ .../cryptonote_protocol_handler-base.cpp | 266 +++++++++++++++++++++ .../cryptonote_protocol_handler.h | 35 ++- .../cryptonote_protocol_handler.inl | 73 +++++- 4 files changed, 408 insertions(+), 12 deletions(-) create mode 100644 src/cryptonote_protocol/CMakeLists.txt create mode 100644 src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp (limited to 'src/cryptonote_protocol') diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt new file mode 100644 index 000000000..2ea5662a1 --- /dev/null +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB CRYPTONOTE_PROTOCOL *) +source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${CRYPTONOTE_PROTOCOL}) +bitmonero_add_library(cryptonote_protocol ${CRYPTONOTE_PROTOCOL}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(cryptonote_protocol + version) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp new file mode 100644 index 000000000..6b25cb681 --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -0,0 +1,266 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include +#include +#include + + +#include +#include + +#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h" +#include "../../src/p2p/network_throttle.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +class cryptonote_protocol_handler_base_pimpl { // placeholer if needed + public: + +}; + +} // namespace + +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa + const double size_min = 500; // XXX 500 + const int history_len = 20; // how many blocks to average over + + double avg=0; + try { + avg = get_avg_block_size(history_len); + } catch (...) { } + avg = std::max( size_min , avg); + return avg; +} + +cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { +} + +cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { +} + +void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) { + using namespace epee::net_utils; + size_t est_req_size=0; // how much data are we now requesting (to be soon send to us) + + const auto count_limit_default = count_limit; + + bool allowed_now = false; // are we now allowed to request or are we limited still + // long int size_limit; + + while (!allowed_now) { + /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit + _fact("ABORT sleep (before sending requeset) due to stopping"); + break; + }*/ + + //LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0); + //{ + long int size_limit1=0, size_limit2=0; + //LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0); + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().tick(); + size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport(); + } + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().tick(); + size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport(); + } + + long int one_block_estimated_size = estimate_one_block_size(); + long int limit_small = std::min( size_limit1 , size_limit2 ); + long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3; + if (limit_small <= 0) size_limit = 0; + const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread + const double knob = 1.000; + size_limit /= (estimated_peers / estimated_peers) * knob; + _note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<& ids) { + using namespace epee::net_utils; + LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0); + _note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + // TODO +} + +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_mark(""); + using namespace epee::net_utils; + double delay=0; // will be calculated + _dbg1("Packet size: " << packet_size); + do + { // rate limiting + //XXX + /*if (::cryptonote::core::get_is_stopping()) { + _dbg1("We are stopping - so abort sleep"); + return; + }*/ + /*if (m_was_shutdown) { + _dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep"); + return; + }*/ + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + + delay *= 0.50; + //delay = 0; // XXX + if (delay > 0) { + //delay += rand2*0.1; + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="< 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global + //epee::critical_region_t guard(m_throttle_global_lock); // *** critical *** + //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + } +} + +} // namespace + + diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index a3b79856e..b3393928c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -1,3 +1,7 @@ +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + // Copyright (c) 2014-2015, The Monero Project // // All rights reserved. @@ -41,6 +45,7 @@ #include "cryptonote_core/connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" +#include PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -48,8 +53,26 @@ DISABLE_VS_WARNINGS(4355) namespace cryptonote { + class cryptonote_protocol_handler_base_pimpl; + class cryptonote_protocol_handler_base { + private: + std::unique_ptr mI; + + public: + cryptonote_protocol_handler_base(); + virtual ~cryptonote_protocol_handler_base(); + void handler_request_blocks_now(size_t & count_limit); // before asking for blocks, can adjust the limit of download + void handler_request_blocks_history(std::list& ids); // before asking for list of objects, we can change the list still + void handler_response_blocks_now(size_t packet_size); + + virtual double get_avg_block_size( size_t count) const = 0; + virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download + + virtual std::ofstream& get_logreq() const =0; + }; + template - class t_cryptonote_protocol_handler: public i_cryptonote_protocol + class t_cryptonote_protocol_handler: public i_cryptonote_protocol, cryptonote_protocol_handler_base { public: typedef cryptonote_connection_context connection_context; @@ -107,12 +130,17 @@ namespace cryptonote std::atomic m_syncronized_connections_count; std::atomic m_synchronized; + // static std::ofstream m_logreq; + + double get_avg_block_size(size_t count) const; + template bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parametr).name() << " -->"); std::string blob; epee::serialization::store_t_to_binary(arg, blob); + //handler_response_blocks_now(blob.size()); // XXX return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context); } @@ -124,8 +152,11 @@ namespace cryptonote epee::serialization::store_t_to_binary(arg, arg_buff); return m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context); } + + virtual std::ofstream& get_logreq() const ; }; -} + +} // namespace #include "cryptonote_protocol_handler.inl" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2754eb73c..aebfcd10d 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1,3 +1,7 @@ +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + // Copyright (c) 2014-2015, The Monero Project // // All rights reserved. @@ -28,14 +32,26 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +// (may contain code and/or modifications by other developers) +// developer rfree: this code is caller of our new network code, and is modded; e.g. for rate limiting + #include #include #include "cryptonote_core/cryptonote_format_utils.h" #include "profile_tools.h" +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + namespace cryptonote { + +// static +// template std::ofstream t_cryptonote_protocol_handler::m_logreq("logreq.txt"); // static + + + //----------------------------------------------------------------------------------------------------------------------- template t_cryptonote_protocol_handler::t_cryptonote_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint* p_net_layout):m_core(rcore), @@ -100,20 +116,24 @@ namespace cryptonote { std::stringstream ss; - ss << std::setw(25) << std::left << "Remote Host" + ss << std::setw(30) << std::left << "Remote Host" << std::setw(20) << "Peer id" << std::setw(25) << "Recv/Sent (inactive,sec)" << std::setw(25) << "State" << std::setw(20) << "Livetime(seconds)" << ENDL; + uint32_t ip; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { - ss << std::setw(25) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + - epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) + ip = ntohl(cntxt.m_remote_ip); + ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + + epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) - << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) << ENDL; + << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) + << std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "") //TODO: local ip in calss A, B + << ENDL; return true; }); LOG_PRINT_L0("Connections: " << ENDL << ss.str()); @@ -234,11 +254,11 @@ namespace cryptonote block_verification_context bvc = boost::value_initialized(); m_core.pause_mine(); - m_core.handle_incoming_block(arg.b.block, bvc); + m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block m_core.resume_mine(); if(bvc.m_verifivation_failed) { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + LOG_PRINT_CCONTEXT_L0("Block verification failed, dropping connection"); m_p2p->drop_connection(context); return 1; } @@ -304,9 +324,19 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify(rsp, context); + //handler_response_blocks_now(sizeof(rsp)); // XXX + //handler_response_blocks_now(200); return 1; } //------------------------------------------------------------------------------------------------------------------------ + + + template + double t_cryptonote_protocol_handler::get_avg_block_size( size_t count) const { + return m_core.get_blockchain_storage().get_avg_block_size(count); + } + + template int t_cryptonote_protocol_handler::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { @@ -378,6 +408,7 @@ namespace cryptonote epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); + LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0); BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) { //process transactions @@ -419,7 +450,8 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); } } - + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + handler_request_blocks_now(count_limit); // XXX request_missing_objects(context, true); return 1; } @@ -455,6 +487,11 @@ namespace cryptonote size_t count = 0; auto it = context.m_needed_objects.begin(); + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + //handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX + // XXX + count_limit=200; // XXX + _note_c("net/req-calc" , "Setting count_limit: " << count_limit); while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) { if( !(check_having_blocks && m_core.have_block(*it))) @@ -465,14 +502,16 @@ namespace cryptonote } context.m_needed_objects.erase(it++); } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()); + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + << "requested blocks count=" << count << " / " << count_limit); post_notify(req, context); }else if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify(r, context); }else { @@ -575,4 +614,18 @@ namespace cryptonote { return relay_post_notify(arg, exclude_context); } -} + + /// @deprecated + template std::ofstream& t_cryptonote_protocol_handler::get_logreq() const { + static std::ofstream * logreq=NULL; + if (!logreq) { + LOG_PRINT_RED("LOG OPENED",LOG_LEVEL_0); + logreq = new std::ofstream("logreq.txt"); // leak mem (singleton) + *logreq << "Opened log" << std::endl; + } + LOG_PRINT_YELLOW("LOG USED",LOG_LEVEL_0); + (*logreq) << "log used" << std::endl; + return *logreq; + } + +} // namespace -- cgit v1.2.3