diff options
Diffstat (limited to 'src/cryptonote_protocol')
3 files changed, 139 insertions, 113 deletions
diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index 6b25cb681..b5a5ceea9 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -104,11 +104,11 @@ namespace cryptonote { double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa const double size_min = 500; // XXX 500 - const int history_len = 20; // how many blocks to average over + //const int history_len = 20; // how many blocks to average over double avg=0; try { - avg = get_avg_block_size(history_len); + avg = get_avg_block_size(/*history_len*/); } catch (...) { } avg = std::max( size_min , avg); return avg; @@ -120,96 +120,6 @@ cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { } -void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) { - using namespace epee::net_utils; - size_t est_req_size=0; // how much data are we now requesting (to be soon send to us) - - const auto count_limit_default = count_limit; - - bool allowed_now = false; // are we now allowed to request or are we limited still - // long int size_limit; - - while (!allowed_now) { - /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO fast exit - _fact("ABORT sleep (before sending requeset) due to stopping"); - break; - }*/ - - //LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0); - //{ - long int size_limit1=0, size_limit2=0; - //LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0); - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); - network_throttle_manager::get_global_throttle_in().tick(); - size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport(); - } - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); - network_throttle_manager::get_global_throttle_inreq().tick(); - size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport(); - } - - long int one_block_estimated_size = estimate_one_block_size(); - long int limit_small = std::min( size_limit1 , size_limit2 ); - long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3; - if (limit_small <= 0) size_limit = 0; - const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread - const double knob = 1.000; - size_limit /= (estimated_peers / estimated_peers) * knob; - _note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<<size_limit); - - double L = size_limit / one_block_estimated_size; // calculating item limit (some heuristics) - //LOG_PRINT_RED("L1 = " << L , LOG_LEVEL_0); - //double L2=0; if (L>1) L2=std::log(L); - //L = L/10. + L2*5; - //LOG_PRINT_RED("L2 = " << L , LOG_LEVEL_0); - L = std::min( (double)count_limit_default, (double)L); - //LOG_PRINT_RED("L3 = " << L , LOG_LEVEL_0); - - const long int hard_limit = 500; // never get more blocks at once ; TODO depend on speed limit. Must be low or limiting is too bursty. - - L = std::min(L, (double) hard_limit); - - count_limit = (int)L; - - est_req_size = count_limit * one_block_estimated_size ; // how much data did we just requested? - - //LOG_PRINT_RED("est_req_size = " << est_req_size , LOG_LEVEL_0); - //LOG_PRINT_RED("count_limit = " << count_limit , LOG_LEVEL_0); - //LOG_PRINT_RED("one_block_estimated_size = " << one_block_estimated_size , LOG_LEVEL_0); - //} - - if (count_limit > 0) allowed_now = true; - // XXX if (!allowed_now) { // XXX DOWNLOAD - //long int ms = 3000; // XXX 2000 - //LOG_PRINT_RED("size_limit = " << size_limit , LOG_LEVEL_0); - long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick(one_block_estimated_size); // XXX too long - //long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time(count_limit); // XXX - //long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time(size_limit); // XXX best - - //ms /= 100; // XXX - _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms"); // XXX debug sleep - //LOG_PRINT_RED("ms = " << ms , LOG_LEVEL_0); - boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps - //} - } - // done waiting&sleeping ^ - - // ok we are allowed to send now - { - CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); - network_throttle_manager::get_global_throttle_inreq().handle_trafic_tcp( est_req_size ); // increase countere of the global requested input - } - - // TODO remove debug - LOG_PRINT_YELLOW("*************************************************************************", LOG_LEVEL_0); - LOG_PRINT_RED("### RRRR ### sending request (type 1), CALCULATED limit = " << count_limit << " = estimated " << est_req_size << " b", LOG_LEVEL_0); - LOG_PRINT_YELLOW("*************************************************************************", LOG_LEVEL_0); - LOG_PRINT_RED("\n", LOG_LEVEL_0); - _note_c("net/req", "### RRRR ### sending request (type 1), CALCULATED limit = " << count_limit << " = estimated " << est_req_size << " b"); -} - void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) { using namespace epee::net_utils; LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index b3393928c..0be864dfa 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -46,6 +46,7 @@ #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" #include <netinet/in.h> +#include <boost/circular_buffer.hpp> PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) @@ -61,11 +62,10 @@ namespace cryptonote public: cryptonote_protocol_handler_base(); virtual ~cryptonote_protocol_handler_base(); - void handler_request_blocks_now(size_t & count_limit); // before asking for blocks, can adjust the limit of download void handler_request_blocks_history(std::list<crypto::hash>& ids); // before asking for list of objects, we can change the list still void handler_response_blocks_now(size_t packet_size); - virtual double get_avg_block_size( size_t count) const = 0; + virtual double get_avg_block_size() = 0; virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download virtual std::ofstream& get_logreq() const =0; @@ -129,10 +129,12 @@ namespace cryptonote nodetool::i_p2p_endpoint<connection_context>* m_p2p; std::atomic<uint32_t> m_syncronized_connections_count; std::atomic<bool> m_synchronized; + bool m_one_request = true; // static std::ofstream m_logreq; - - double get_avg_block_size(size_t count) const; + std::mutex m_buffer_mutex; + double get_avg_block_size(); + boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10); template<class t_parametr> bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index aebfcd10d..0130cb384 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -41,6 +41,7 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "profile_tools.h" #include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/network_throttle-detail.hpp" using namespace nOT::nUtils; namespace cryptonote @@ -115,28 +116,66 @@ namespace cryptonote void t_cryptonote_protocol_handler<t_core>::log_connections() { std::stringstream ss; + ss.precision(1); + + double down_sum = 0.0; + double down_curr_sum = 0.0; + double up_sum = 0.0; + double up_curr_sum = 0.0; ss << std::setw(30) << std::left << "Remote Host" << std::setw(20) << "Peer id" - << std::setw(25) << "Recv/Sent (inactive,sec)" + << std::setw(30) << "Recv/Sent (inactive,sec)" << std::setw(25) << "State" - << std::setw(20) << "Livetime(seconds)" << ENDL; + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << ENDL; uint32_t ip; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { + bool local_ip = false; ip = ntohl(cntxt.m_remote_ip); + // TODO: local ip in calss A, B + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + local_ip = true; + auto connection_time = time(NULL) - cntxt.m_started; ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id - << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" + << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) - << std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "") //TODO: local ip in calss A, B - << ENDL; + << std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024) + << std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024 + << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024) + << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024 + << (local_ip ? "[LAN]" : "") + << std::left << (ip == 2130706433 ? "[LOCALHOST]" : "") // 127.0.0.1 + << ENDL; + + if (connection_time > 0) + { + down_sum += (cntxt.m_recv_cnt / connection_time / 1024); + up_sum += (cntxt.m_send_cnt / connection_time / 1024); + } + + down_curr_sum += (cntxt.m_current_speed_down / 1024); + up_curr_sum += (cntxt.m_current_speed_up / 1024); + return true; }); - LOG_PRINT_L0("Connections: " << ENDL << ss.str()); + ss << ENDL + << std::setw(125) << " " + << std::setw(12) << down_sum + << std::setw(14) << down_curr_sum + << std::setw(10) << up_sum + << std::setw(13) << up_curr_sum + << ENDL; + LOG_PRINT_L0("Connections: " << ENDL << ss.str()); } //------------------------------------------------------------------------------------------------------------------------ // Returns a list of connection_info objects describing each open p2p connection @@ -332,8 +371,22 @@ namespace cryptonote template<class t_core> - double t_cryptonote_protocol_handler<t_core>::get_avg_block_size( size_t count) const { - return m_core.get_blockchain_storage().get_avg_block_size(count); + double t_cryptonote_protocol_handler<t_core>::get_avg_block_size() { + // return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download + + CRITICAL_REGION_LOCAL(m_buffer_mutex); + double avg = 0; + if (m_avg_buffer.size() == 0) { + _warn("m_avg_buffer.size() == 0"); + return 500; + } + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + for (auto element : m_avg_buffer) avg += element; + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + return avg / m_avg_buffer.size(); } @@ -341,6 +394,41 @@ namespace cryptonote int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS"); + + // calculate size of request - mainly for logging/debug + size_t size = 0; + for (auto element : arg.txs) + size += element.size(); + + for (auto element : arg.blocks) + { + size += element.block.size(); + for (auto tx : element.txs) + size += tx.size(); + } + + for (auto element : arg.missed_ids) + size += sizeof(element.data); + + size += sizeof(arg.current_blockchain_height); + { + CRITICAL_REGION_LOCAL(m_buffer_mutex); + m_avg_buffer.push_back(size); + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + m_avg_buffer.push_back(666); // a test value + m_avg_buffer.erase_end(1); + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + } + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + if(context.m_last_response_height > arg.current_blockchain_height) { LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height @@ -430,8 +518,9 @@ namespace cryptonote //process block TIME_MEASURE_START(block_process_time); block_verification_context bvc = boost::value_initialized<block_verification_context>(); - - m_core.handle_incoming_block(block_entry.block, bvc, false); + + if (m_core.get_check_blocks()) + m_core.handle_incoming_block(block_entry.block, bvc, false); if(bvc.m_verifivation_failed) { @@ -448,10 +537,21 @@ namespace cryptonote TIME_MEASURE_FINISH(block_process_time); LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); + + std::ofstream log_file; + log_file.open("log/dr-monero/get_objects_calc_time.data", std::ofstream::out | std::ofstream::app); + log_file.precision(7); + + using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count(); + double ms_f = m_ms; + ms_f /= 1000.; + + log_file << static_cast<int>(ms_f) << " " << block_process_time + transactions_process_time << std::endl; } } - size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; - handler_request_blocks_now(count_limit); // XXX request_missing_objects(context, true); return 1; } @@ -480,6 +580,15 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks) { + //if (!m_one_request == false) + //return true; + m_one_request = false; + // save request size to log (dr monero) + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + if(context.m_needed_objects.size()) { //we know objects that we need, request this objects @@ -487,11 +596,8 @@ namespace cryptonote size_t count = 0; auto it = context.m_needed_objects.begin(); - size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; - //handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX - // XXX - count_limit=200; // XXX - _note_c("net/req-calc" , "Setting count_limit: " << count_limit); + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + _note_c("net/req-calc" , "Setting count_limit: " << count_limit); while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) { if( !(check_having_blocks && m_core.have_block(*it))) @@ -504,6 +610,8 @@ namespace cryptonote } LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() << "requested blocks count=" << count << " / " << count_limit); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); }else if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry @@ -511,6 +619,12 @@ namespace cryptonote NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + + //std::string blob; // for calculate size of request + //epee::serialization::store_t_to_binary(r, blob); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + LOG_PRINT_CCONTEXT_L0("r = " << 200); + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); }else |