2014 network limit 1.1 +utils +toc -doc -drmonero

Update of the PR with network limits

works very well for all speeds
(but remember that low download speed can stop upload
because we then slow down downloading of blockchain
requests too)

more debug options

fixed pedantic warnings in our code
should work again on Mac OS X and FreeBSD
fixed warning about size_t
tested on Debian, Ubuntu, Windows(testing now)

TCP options and ToS (QoS) flag
FIXED peer number limit
FIXED some spikes in ingress/download
FIXED problems when other up and down limit

3 files changed, 139 insertions, 113 deletions

diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp
index 6b25cb681..b5a5ceea9 100644
--- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp
+++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp
@@ -104,11 +104,11 @@ namespace cryptonote {
 
 double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa
  const double size_min = 500; // XXX 500
- const int history_len = 20; // how many blocks to average over
+ //const int history_len = 20; // how many blocks to average over
 
  double avg=0;
  try {
- avg = get_avg_block_size(history_len);
+ avg = get_avg_block_size(/*history_len*/);
  } catch (...) { }
  avg = std::max( size_min , avg);
  return avg;
@@ -120,96 +120,6 @@ cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() {
 cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() {
 }
 
-void cryptonote_protocol_handler_base::handler_request_blocks_now(size_t &count_limit) {
-	using namespace epee::net_utils;
-	size_t est_req_size=0; //  how much data are we now requesting (to be soon send to us)
-
-	const auto count_limit_default = count_limit;
-
-	bool allowed_now = false; // are we now allowed to request or are we limited still
-	// long int size_limit;
-
-	while (!allowed_now) {
-		/* if ( ::cryptonote::core::get_is_stopping() ) {  // TODO fast exit
-			_fact("ABORT sleep (before sending requeset) due to stopping"); 
-			break;
-		}*/
-
-			//LOG_PRINT_RED("[DBG]" << get_avg_block_size(1), LOG_LEVEL_0);
-		//{
-			long int size_limit1=0, size_limit2=0;
-			//LOG_PRINT_RED("calculating REQUEST size:", LOG_LEVEL_0);
-			{
-				CRITICAL_REGION_LOCAL(	network_throttle_manager::m_lock_get_global_throttle_in );
-				network_throttle_manager::get_global_throttle_in().tick();
-				size_limit1 = network_throttle_manager::get_global_throttle_in().get_recommended_size_of_planned_transport();
-			}
-			{
-				CRITICAL_REGION_LOCAL(	network_throttle_manager::m_lock_get_global_throttle_inreq );
-				network_throttle_manager::get_global_throttle_inreq().tick();
-				size_limit2 = network_throttle_manager::get_global_throttle_inreq().get_recommended_size_of_planned_transport();
-			}
-
-			long int one_block_estimated_size = estimate_one_block_size();
-			long int limit_small = std::min( size_limit1 , size_limit2 );
-			long int size_limit = limit_small/3 + size_limit1/3 + size_limit2/3;
-			if (limit_small <= 0) size_limit = 0;
-			const double estimated_peers = 1.2; // how many peers/threads we want to talk to, in order to not grab entire b/w by 1 thread
-			const double knob = 1.000;
-			size_limit /= (estimated_peers / estimated_peers) * knob;
-			_note_c("net/req-calc" , "calculating REQUEST size:" << size_limit1 << " " << size_limit2 << " small=" << limit_small << " final size_limit="<<size_limit);
-
-			double L = size_limit / one_block_estimated_size; // calculating item limit (some heuristics)
-			//LOG_PRINT_RED("L1 = " << L , LOG_LEVEL_0);
-			//double L2=0; if (L>1) L2=std::log(L);
-			//L = L/10. + L2*5;
-			//LOG_PRINT_RED("L2 = " << L , LOG_LEVEL_0);
-			L = std::min( (double)count_limit_default, (double)L);
-			//LOG_PRINT_RED("L3 = " << L , LOG_LEVEL_0);
-
-			const long int hard_limit = 500; // never get more blocks at once ; TODO depend on speed limit. Must be low or limiting is too bursty.
-
-			L = std::min(L, (double) hard_limit);
-
-			count_limit = (int)L;
-
-			est_req_size = count_limit * one_block_estimated_size ; // how much data did we just requested?
-			
-			//LOG_PRINT_RED("est_req_size = " << est_req_size , LOG_LEVEL_0);
-			//LOG_PRINT_RED("count_limit = " << count_limit , LOG_LEVEL_0);
-			//LOG_PRINT_RED("one_block_estimated_size = " << one_block_estimated_size , LOG_LEVEL_0);
-		//}
-
-		if (count_limit > 0) allowed_now = true;
-       // XXX if (!allowed_now) { // XXX DOWNLOAD
-			//long int ms = 3000; // XXX 2000
-			//LOG_PRINT_RED("size_limit = " << size_limit , LOG_LEVEL_0);
-			long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick(one_block_estimated_size); // XXX too long
-			//long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time(count_limit); // XXX
-			//long int ms = network_throttle_manager::get_global_throttle_in().get_sleep_time(size_limit); // XXX best
-			
-			//ms /= 100; // XXX
-			_info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms"); // XXX debug sleep
-			//LOG_PRINT_RED("ms = " << ms , LOG_LEVEL_0);
-			boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps
-		//}
-	}
-	// done waiting&sleeping ^
-
-	// ok we are allowed to send now
-	{
-		CRITICAL_REGION_LOCAL(	network_throttle_manager::m_lock_get_global_throttle_inreq );
-		network_throttle_manager::get_global_throttle_inreq().handle_trafic_tcp( est_req_size ); // increase countere of the global requested input
-	}
-
-	// TODO remove debug
-	LOG_PRINT_YELLOW("*************************************************************************", LOG_LEVEL_0);
-	LOG_PRINT_RED("### RRRR ### sending request (type 1), CALCULATED limit = " << count_limit << " = estimated " << est_req_size << " b", LOG_LEVEL_0);
-	LOG_PRINT_YELLOW("*************************************************************************", LOG_LEVEL_0);
-	LOG_PRINT_RED("\n", LOG_LEVEL_0);
-	_note_c("net/req", "### RRRR ### sending request (type 1), CALCULATED limit = " << count_limit << " = estimated " << est_req_size << " b");
-}
-
 void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) {
 	using namespace epee::net_utils;
 	LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size());
diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h
index b3393928c..0be864dfa 100644
--- a/src/cryptonote_protocol/cryptonote_protocol_handler.h
+++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h
@@ -46,6 +46,7 @@
 #include "cryptonote_core/cryptonote_stat_info.h"
 #include "cryptonote_core/verification_context.h"
 #include <netinet/in.h>
+#include <boost/circular_buffer.hpp>
 
 PUSH_WARNINGS
 DISABLE_VS_WARNINGS(4355)
@@ -61,11 +62,10 @@ namespace cryptonote
 		public:
 			cryptonote_protocol_handler_base();
 			virtual ~cryptonote_protocol_handler_base();
-			void handler_request_blocks_now(size_t & count_limit); // before asking for blocks, can adjust the limit of download
 			void handler_request_blocks_history(std::list<crypto::hash>& ids); // before asking for list of objects, we can change the list still
 			void handler_response_blocks_now(size_t packet_size);
 			
-			virtual double get_avg_block_size( size_t count) const = 0;
+			virtual double get_avg_block_size() = 0;
 			virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download
 
 			virtual std::ofstream& get_logreq() const =0;
@@ -129,10 +129,12 @@ namespace cryptonote
     nodetool::i_p2p_endpoint<connection_context>* m_p2p;
     std::atomic<uint32_t> m_syncronized_connections_count;
     std::atomic<bool> m_synchronized;
+    bool m_one_request = true;
 
 		// static std::ofstream m_logreq;
-    
-    double get_avg_block_size(size_t count) const;
+    std::mutex m_buffer_mutex;
+    double get_avg_block_size();
+    boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10);
 
     template<class t_parametr>
       bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context)
diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl
index aebfcd10d..0130cb384 100644
--- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl
+++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl
@@ -41,6 +41,7 @@
 #include "cryptonote_core/cryptonote_format_utils.h"
 #include "profile_tools.h"
 #include "../../contrib/otshell_utils/utils.hpp"
+#include "../../src/p2p/network_throttle-detail.hpp"
 using namespace nOT::nUtils;
 
 namespace cryptonote
@@ -115,28 +116,66 @@ namespace cryptonote
   void t_cryptonote_protocol_handler<t_core>::log_connections()
   {
     std::stringstream ss;
+    ss.precision(1);
+    
+    double down_sum			= 0.0;
+    double down_curr_sum	= 0.0;
+    double up_sum			= 0.0;
+    double up_curr_sum		= 0.0;
 
     ss << std::setw(30) << std::left << "Remote Host"
       << std::setw(20) << "Peer id"
-      << std::setw(25) << "Recv/Sent (inactive,sec)"
+      << std::setw(30) << "Recv/Sent (inactive,sec)"
       << std::setw(25) << "State"
-      << std::setw(20) << "Livetime(seconds)" << ENDL;
+      << std::setw(20) << "Livetime(sec)"
+      << std::setw(12) << "Down (kB/s)"
+      << std::setw(14) << "Down(now)"
+      << std::setw(10) << "Up (kB/s)" 
+      << std::setw(13) << "Up(now)"
+      << ENDL;
 
 	uint32_t ip;
     m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id)
     {
+	  bool local_ip = false;
 	  ip = ntohl(cntxt.m_remote_ip);
+	  // TODO: local ip in calss A, B
+	  if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x
+		local_ip = true;
+	  auto connection_time = time(NULL) - cntxt.m_started;
       ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") +
         epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port)
         << std::setw(20) << std::hex << peer_id
-        << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
+        << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")"
         << std::setw(25) << get_protocol_state_string(cntxt.m_state)
         << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started)
-        << std::setw(10) << (ip > 3232235520 && ip < 3232301055 ? " [LAN]" : "")		//TODO: local ip in calss A, B
-        << ENDL; 
+        << std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024)
+        << std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024
+        << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024)
+        << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024
+        << (local_ip ? "[LAN]" : "")
+        << std::left << (ip == 2130706433 ? "[LOCALHOST]" : "") // 127.0.0.1
+        << ENDL;
+        
+        if (connection_time > 0)
+        {
+			down_sum += (cntxt.m_recv_cnt / connection_time / 1024);
+			up_sum += (cntxt.m_send_cnt / connection_time / 1024);
+		}
+		
+		down_curr_sum += (cntxt.m_current_speed_down / 1024);
+		up_curr_sum += (cntxt.m_current_speed_up / 1024);
+        
       return true;
     });
-    LOG_PRINT_L0("Connections: " << ENDL << ss.str());
+	ss << ENDL
+	  << std::setw(125) << " "
+	  << std::setw(12) << down_sum
+	  << std::setw(14) << down_curr_sum
+	  << std::setw(10) << up_sum
+	  << std::setw(13) << up_curr_sum
+	<< ENDL;
+	LOG_PRINT_L0("Connections: " << ENDL << ss.str());
   }
   //------------------------------------------------------------------------------------------------------------------------   
   // Returns a list of connection_info objects describing each open p2p connection
@@ -332,8 +371,22 @@ namespace cryptonote
 
   
   template<class t_core>
-  double t_cryptonote_protocol_handler<t_core>::get_avg_block_size( size_t count) const {
-		return m_core.get_blockchain_storage().get_avg_block_size(count);
+  double t_cryptonote_protocol_handler<t_core>::get_avg_block_size() {
+		// return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download
+
+		CRITICAL_REGION_LOCAL(m_buffer_mutex);
+		double avg = 0;
+		if (m_avg_buffer.size() == 0) {
+			_warn("m_avg_buffer.size() == 0");
+			return 500;
+		}
+
+		const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option
+		long int dbg_repeat=0;
+		do {
+			for (auto element : m_avg_buffer) avg += element;
+		} while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one
+		return avg / m_avg_buffer.size();
   }
 
   
@@ -341,6 +394,41 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context)
   {
     LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS");
+    
+    // calculate size of request - mainly for logging/debug
+    size_t size = 0;
+    for (auto element : arg.txs)
+		size += element.size();
+	
+	for (auto element : arg.blocks)
+	{
+		size += element.block.size();
+		for (auto tx : element.txs)
+			size += tx.size();
+	}
+	
+	for (auto element : arg.missed_ids)
+		size += sizeof(element.data);
+	
+	size += sizeof(arg.current_blockchain_height);
+	{
+		CRITICAL_REGION_LOCAL(m_buffer_mutex);
+		m_avg_buffer.push_back(size);
+
+		const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option
+		long int dbg_repeat=0;
+		do {
+			m_avg_buffer.push_back(666); // a test value
+			m_avg_buffer.erase_end(1);
+		} while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one
+	}
+	/*using namespace boost::chrono;
+	auto point = steady_clock::now();
+	auto time_from_epoh = point.time_since_epoch();
+	auto sec = duration_cast< seconds >( time_from_epoh ).count();*/
+	
+	//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
+	
     if(context.m_last_response_height > arg.current_blockchain_height)
     {
       LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height 
@@ -430,8 +518,9 @@ namespace cryptonote
         //process block
         TIME_MEASURE_START(block_process_time);
         block_verification_context bvc = boost::value_initialized<block_verification_context>();
-
-        m_core.handle_incoming_block(block_entry.block, bvc, false);
+		
+		if (m_core.get_check_blocks())
+			m_core.handle_incoming_block(block_entry.block, bvc, false);
 
         if(bvc.m_verifivation_failed)
         {
@@ -448,10 +537,21 @@ namespace cryptonote
 
         TIME_MEASURE_FINISH(block_process_time);
         LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms");
+        
+        std::ofstream log_file;
+		log_file.open("log/dr-monero/get_objects_calc_time.data", std::ofstream::out | std::ofstream::app);
+		log_file.precision(7);
+
+		using namespace boost::chrono;
+		auto point = steady_clock::now();
+		auto time_from_epoh = point.time_since_epoch();
+		auto m_ms = duration_cast< milliseconds >( time_from_epoh ).count();
+		double ms_f = m_ms;
+		ms_f /= 1000.;
+		
+		log_file << static_cast<int>(ms_f) << " " << block_process_time + transactions_process_time << std::endl;
       }
     }
-    size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
-	handler_request_blocks_now(count_limit); // XXX
     request_missing_objects(context, true);
     return 1;
   }
@@ -480,6 +580,15 @@ namespace cryptonote
   template<class t_core> 
   bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks)
   {
+	//if (!m_one_request == false)
+		//return true;
+	m_one_request = false;
+  	// save request size to log (dr monero)
+		/*using namespace boost::chrono;
+		auto point = steady_clock::now();
+		auto time_from_epoh = point.time_since_epoch();
+		auto sec = duration_cast< seconds >( time_from_epoh ).count();*/
+		
     if(context.m_needed_objects.size())
     {
       //we know objects that we need, request this objects
@@ -487,11 +596,8 @@ namespace cryptonote
       size_t count = 0;
       auto it = context.m_needed_objects.begin();
 
-			size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
-            //handler_request_blocks_now( count_limit ); // change the limit, sleep(?) XXX
-     		// XXX 
-            	count_limit=200; // XXX
-		_note_c("net/req-calc" , "Setting count_limit: " << count_limit);
+	  size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT;
+	  _note_c("net/req-calc" , "Setting count_limit: " << count_limit);
       while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT)
       {
         if( !(check_having_blocks && m_core.have_block(*it)))
@@ -504,6 +610,8 @@ namespace cryptonote
       }
       LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()
 				<< "requested blocks count=" << count << " / " << count_limit);
+		//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
+		
       post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);    
     }else if(context.m_last_response_height < context.m_remote_blockchain_height-1)
     {//we have to fetch more objects ids, request blockchain entry
@@ -511,6 +619,12 @@ namespace cryptonote
       NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>();
       m_core.get_short_chain_history(r.block_ids);
 		 	handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
+		
+		//std::string blob; // for calculate size of request
+		//epee::serialization::store_t_to_binary(r, blob);
+		//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
+		LOG_PRINT_CCONTEXT_L0("r = " << 200);
+		
       LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
     }else