2014 network limit 1.1 +utils +toc -doc -drmonero

Update of the PR with network limits

works very well for all speeds
(but remember that low download speed can stop upload
because we then slow down downloading of blockchain
requests too)

more debug options

fixed pedantic warnings in our code
should work again on Mac OS X and FreeBSD
fixed warning about size_t
tested on Debian, Ubuntu, Windows(testing now)

TCP options and ToS (QoS) flag
FIXED peer number limit
FIXED some spikes in ingress/download
FIXED problems when other up and down limit

1 files changed, 131 insertions, 17 deletions

diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl
index aebfcd10d..0130cb384 100644
--- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl
+++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl
@@ -41,6 +41,7 @@
 #include "cryptonote_core/cryptonote_format_utils.h"
 #include "profile_tools.h"
 #include "../../contrib/otshell_utils/utils.hpp"
+#include "../../src/p2p/network_throttle-detail.hpp"
 using namespace nOT::nUtils;
 
 namespace cryptonote
@@ -115,28 +116,66 @@ namespace cryptonote
   void t_cryptonote_protocol_handler<t_core>::log_connections()
   {
     std::stringstream ss;
+    ss.precision(1);
+    
+    double down_sum			= 0.0;
+    double down_curr_sum	= 0.0;
+    double up_sum			= 0.0;
+    double up_curr_sum		= 0.0;
 
     ss << std::setw(30) << std::left << "Remote Host"
       << std::setw(20) << "Peer id"
-      << std::setw(25) << "Recv/Sent (inactive,sec)"
+      << std::setw(30) << "Recv/Sent (inactive,sec)"
       << std::setw(25) << "State"
-      << std::setw(20) << "Livetime(seconds)" << ENDL;
+      << std::setw(20) << "Livetime(sec)"
+      << std::setw(12) << "Down (kB/s)"
+      << std::setw(14) << "Down(now)"
+      << std::setw(10) << "Up (kB/s)" 
+      << std::setw(13) << "Up(now)"
+      << ENDL;
 
 	uint32_t ip;
     m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id)
     {
+	  bool local_ip = false;
 	  ip = ntohl(cntxt.m_remote_ip);
+	  // TODO: local ip in calss A, B
+	  if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x
+		local_ip = true;
+	  auto connection_time = time(NULL) - cntxt.m_started;
       ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") +
         epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port)
         << std::setw(20) << std::hex << peer_id
-        << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
+        << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
         << std::setw(25) << get_protocol_state_string(cntxt.m_state)
         << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started)
-        << std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "")		//TODO: local ip in calss A, B
-        << ENDL; 
+        << std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024)
+        << std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024
+        << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024)
+        << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024
+        << (local_ip ? "[LAN]" : "")
+        << std::left << (ip == 2130706433 ? "[LOCALHOST]" : "") // 127.0.0.1
+        << ENDL;
+        
+        if (connection_time > 0)
+        {
+			down_sum += (cntxt.m_recv_cnt / connection_time / 1024);
+			up_sum += (cntxt.m_send_cnt / connection_time / 1024);
+		}
+		
+		down_curr_sum += (cntxt.m_current_speed_down / 1024);
+		up_curr_sum += (cntxt.m_current_speed_up / 1024);
+        
       return true;
     });
-    LOG_PRINT_L0("Connections: " << ENDL << ss.str());
+	ss << ENDL
+	  << std::setw(125) << " "
+	  << std::setw(12) << down_sum
+	  << std::setw(14) << down_curr_sum
+	  << std::setw(10) << up_sum
+	  << std::setw(13) << up_curr_sum
+	<< ENDL;
+	LOG_PRINT_L0("Connections: " << ENDL << ss.str());
   }
   //------------------------------------------------------------------------------------------------------------------------   
   // Returns a list of connection_info objects describing each open p2p connection
@@ -332,8 +371,22 @@ namespace cryptonote
 
   
   template<class t_core>
-  double t_cryptonote_protocol_handler<t_core>::get_avg_block_size( size_t count) const {
-		return m_core.get_blockchain_storage().get_avg_block_size(count);
+  double t_cryptonote_protocol_handler<t_core>::get_avg_block_size() {
+		// return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download
+
+		CRITICAL_REGION_LOCAL(m_buffer_mutex);
+		double avg = 0;
+		if (m_avg_buffer.size() == 0) {
+			_warn("m_avg_buffer.size() == 0");
+			return 500;
+		}
+
+		const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option
+		long int dbg_repeat=0;
+		do {
+			for (auto element : m_avg_buffer) avg += element;
+		} while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one
+		return avg / m_avg_buffer.size();
   }
 
   
@@ -341,6 +394,41 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context)
   {
     LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS");
+    
+    // calculate size of request - mainly for logging/debug
+    size_t size = 0;
+    for (auto element : arg.txs)
+		size += element.size();
+	
+	for (auto element : arg.blocks)
+	{
+		size += element.block.size();
+		for (auto tx : element.txs)
+			size += tx.size();
+	}
+	
+	for (auto element : arg.missed_ids)
+		size += sizeof(element.data);
+	
+	size += sizeof(arg.current_blockchain_height);
+	{
+		CRITICAL_REGION_LOCAL(m_buffer_mutex);
+		m_avg_buffer.push_back(size);
+
+		const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option
+		long int dbg_repeat=0;
+		do {
+			m_avg_buffer.push_back(666); // a test value
+			m_avg_buffer.erase_end(1);
+		} while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one
+	}
+	/*using namespace boost::chrono;
+	auto point = steady_clock::now();
+	auto time_from_epoh = point.time_since_epoch();
+	auto sec = duration_cast< seconds >( time_from_epoh ).count();*/
+	
+	//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
+	
     if(context.m_last_response_height > arg.current_blockchain_height)
     {
       LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height 
@@ -430,8 +518,9 @@ namespace cryptonote
         //process block
         TIME_MEASURE_START(block_process_time);
         block_verification_context bvc = boost::value_initialized<block_verification_context>();
-
-        m_core.handle_incoming_block(block_entry.block, bvc, false);
+		
+		if (m_core.get_check_blocks())
+			m_core.handle_incoming_block(block_entry.block, bvc, false);
 
         if(bvc.m_verifivation_failed)
         {
@@ -448,10 +537,21 @@ namespace cryptonote
 
         TIME_MEASURE_FINISH(block_process_time);
         LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms");
+        
+        std::ofstream log_file;
+		log_file.open("log/dr-monero/get_objects_calc_time.data", std::ofstream::out | std::ofstream::app);
+		log_file.precision(7);
+
+		using namespace boost::chrono;
+		auto point = steady_clock::now();
+		auto time_from_epoh = point.time_since_epoch();
+		auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count();
+		double ms_f = m_ms;
+		ms_f /= 1000.;
+		
+		log_file << static_cast<int>(ms_f) << " " << block_process_time + transactions_process_time << std::endl;
       }
     }
-    size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
-	handler_request_blocks_now(count_limit); // XXX
     request_missing_objects(context, true);
     return 1;
   }
@@ -480,6 +580,15 @@ namespace cryptonote
   template<class t_core> 
   bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks)
   {
+	//if (!m_one_request == false)
+		//return true;
+	m_one_request = false;
+  	// save request size to log (dr monero)
+		/*using namespace boost::chrono;
+		auto point = steady_clock::now();
+		auto time_from_epoh = point.time_since_epoch();
+		auto sec = duration_cast< seconds >( time_from_epoh ).count();*/
+		
     if(context.m_needed_objects.size())
     {
       //we know objects that we need, request this objects
@@ -487,11 +596,8 @@ namespace cryptonote
       size_t count = 0;
       auto it = context.m_needed_objects.begin();
 
-			size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
-            //handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX
-     		// XXX 
-            	count_limit=200; // XXX
-		_note_c("net/req-calc" , "Setting count_limit: " << count_limit);
+	  size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
+	  _note_c("net/req-calc" , "Setting count_limit: " << count_limit);
       while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT)
       {
         if( !(check_having_blocks && m_core.have_block(*it)))
@@ -504,6 +610,8 @@ namespace cryptonote
       }
       LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()
 				<< "requested blocks count=" << count << " / " << count_limit);
+		//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
+		
       post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);    
     }else if(context.m_last_response_height < context.m_remote_blockchain_height-1)
     {//we have to fetch more objects ids, request blockchain entry
@@ -511,6 +619,12 @@ namespace cryptonote
       NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>();
       m_core.get_short_chain_history(r.block_ids);
 		 	handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
+		
+		//std::string blob; // for calculate size of request
+		//epee::serialization::store_t_to_binary(r, blob);
+		//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
+		LOG_PRINT_CCONTEXT_L0("r = " << 200);
+		
       LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
     }else