2014 network limit 1.2 +utils +toc -doc -drmonero

new update of the pr with network limits

more debug options:
discarding downloaded blocks all or after given height.
trying to trigger the locking errors.

debug levels polished/tuned to sane values.
debug/logging improved.

warning: this pr should be correct code, but it could make
an existing (in master version) locking error appear more often.

it's a race on the list (map) of peers, e.g. between closing/deleting
them versus working on them in net-limit sleep in sending chunk.

the bug is not in this code/this pr, but in the master version.

the locking problem of master will be fixed in other pr.

problem is ub, and in practice is seems to usually cause program abort
(tested on debian stable with updated gcc). see --help for option
to add sleep to trigger the error faster.

6 files changed, 55 insertions, 52 deletions

diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp
index 0e2fd5942..4a4a32384 100644
--- a/src/p2p/connection_basic.cpp
+++ b/src/p2p/connection_basic.cpp
@@ -152,7 +152,7 @@ connection_basic::connection_basic(boost::asio::io_service& io_service, std::ato
 	try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ;
 
 	_note("Spawned connection p2p#"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count);
-	boost::filesystem::create_directories("log/dr-monero/net/");
+	//boost::filesystem::create_directories("log/dr-monero/net/");
 }
 
 connection_basic::~connection_basic() {
@@ -192,24 +192,15 @@ void connection_basic::save_limit_to_file(int limit) {
     // saving limit to file
     if (!epee::net_utils::data_logger::m_save_graph)
 		return;
-    std::ofstream file_up, file_down;
-    file_up.open("log/dr-monero/limit_up.info", std::ofstream::out | std::ofstream::app);
-    file_up.precision(8);
-    file_down.open("log/dr-monero/limit_down.info", std::ofstream::out | std::ofstream::app);
-    file_down.precision(8);
-    using namespace boost::chrono;
-	auto point = steady_clock::now();
-	auto time_from_epoh = point.time_since_epoch();
-	auto s = duration_cast< seconds >( time_from_epoh ).count();
 
     {
          CRITICAL_REGION_LOCAL(        network_throttle_manager::m_lock_get_global_throttle_out );
-               file_up << s << " " << network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024 << "\n";
+               epee::net_utils::data_logger::get_instance().add_data("upload_limit", network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024);
 	}
 	
     {
          CRITICAL_REGION_LOCAL(        network_throttle_manager::m_lock_get_global_throttle_in );
-               file_down << s << " " << network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024 << "\n";
+               epee::net_utils::data_logger::get_instance().add_data("download_limit", network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024);
 	}
 }
  
diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp
index 6a8eb25be..77f647bed 100644
--- a/src/p2p/data_logger.cpp
+++ b/src/p2p/data_logger.cpp
@@ -1,7 +1,9 @@
 #include "data_logger.hpp"
 
 #include <boost/chrono.hpp>
+#include <boost/filesystem.hpp>
 #include <chrono>
+#include "../../contrib/otshell_utils/utils.hpp"
 
 namespace epee
 {
@@ -32,7 +34,16 @@ namespace net_utils
 		mFilesMap["request"] = data_logger::fileData("log/dr-monero/net/req-all.data");
 		mFilesMap["sleep_down"] = data_logger::fileData("log/dr-monero/down_sleep_log.data");
 		mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data");
+		mFilesMap["calc_time"] = data_logger::fileData("log/dr-monero/get_objects_calc_time.data");
+		mFilesMap["blockchain_processing_time"] = data_logger::fileData("log/dr-monero/blockchain_log.data");
 		
+		mFilesMap["peers_limit"] = data_logger::fileData("log/dr-monero/peers_limit.info");
+		mFilesMap["download_limit"] = data_logger::fileData("log/dr-monero/limit_down.info");
+		mFilesMap["upload_limit"] = data_logger::fileData("log/dr-monero/limit_up.info");
+		
+		mFilesMap["peers_limit"].mLimitFile = true;
+		mFilesMap["download_limit"].mLimitFile = true;
+		mFilesMap["upload_limit"].mLimitFile = true;
 	}
 	
 	void data_logger::add_data(std::string filename, unsigned int data)
@@ -40,7 +51,14 @@ namespace net_utils
 		if (mFilesMap.find(filename) == mFilesMap.end())
 			return; // TODO: exception
 			
-		mFilesMap[filename].mDataToSave += data;
+		
+		nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/");
+		
+		std::lock_guard<std::mutex> lock(mSaveMutex);
+		if (mFilesMap[filename].mLimitFile)
+			mFilesMap[filename].mDataToSave = data;
+		else
+			mFilesMap[filename].mDataToSave += data;
 	}
 	
 	double data_logger::fileData::get_current_time()
@@ -56,23 +74,27 @@ namespace net_utils
 	data_logger::fileData::fileData(std::string pFile)
 	{
 		mFile = std::make_shared<std::ofstream> (pFile);
+		mPath = pFile;
 	}
 	
 	void data_logger::fileData::save()
 	{
 		if (!data_logger::m_save_graph)
 			return;
+		mFile->open(mPath, std::ios::app);
 		*mFile << static_cast<int>(get_current_time()) << " " << mDataToSave << std::endl;
+		mFile->close();
 	}
 	
 	void data_logger::saveToFile()
 	{
-		 std::lock_guard<std::mutex> lock(mSaveMutex);
-		 for (auto &element : mFilesMap)
-		 {
-			 element.second.save();
-			 element.second.mDataToSave = 0;
-		 }
+		std::lock_guard<std::mutex> lock(mSaveMutex);
+		for (auto &element : mFilesMap)
+		{
+			element.second.save();
+			if (!element.second.mLimitFile)
+				element.second.mDataToSave = 0;
+		}
 	}
 	
 std::atomic<bool> data_logger::m_save_graph(false);
diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp
index 2b8503df3..50beb847a 100644
--- a/src/p2p/data_logger.hpp
+++ b/src/p2p/data_logger.hpp
@@ -27,12 +27,15 @@ namespace net_utils
 			{
 				public:
 					fileData(){}
+					fileData(const fileData &ob) = delete;
 					fileData(std::string pFile);
 					
 					std::shared_ptr<std::ofstream> mFile;
 					long int mDataToSave = 0;
 					static double get_current_time();
 					void save();
+					std::string mPath;
+					bool mLimitFile = false;
 			};
 			
 			std::map <std::string, fileData> mFilesMap;
diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h
index ea7d5c383..5417ffa52 100644
--- a/src/p2p/net_node.h
+++ b/src/p2p/net_node.h
@@ -87,7 +87,7 @@ namespace nodetool
 		m_hide_my_port(false),
 		m_network_id(std::move(network_id))
     {
-		m_number_of_out_peers = 0;
+		m_current_number_of_out_peers = 0;
 		m_save_graph = false;
 	}
 
@@ -228,7 +228,7 @@ namespace nodetool
 
 	public:
     config m_config; // TODO was private, add getters?
-    std::atomic<unsigned int> m_number_of_out_peers;
+    std::atomic<unsigned int> m_current_number_of_out_peers;
 	void set_save_graph(bool save_graph)
 	{
 		m_save_graph = save_graph;
diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl
index 60eed1f36..a015763bf 100644
--- a/src/p2p/net_node.inl
+++ b/src/p2p/net_node.inl
@@ -296,20 +296,18 @@ namespace nodetool
 		unsigned int number_of_peers;
 		while (1)
 		{
-			if (m_save_graph)
+			//number_of_peers = m_net_server.get_config_object().get_connections_count();
+			number_of_peers = 0;
+			m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt)
 			{
-				//number_of_peers = m_net_server.get_config_object().get_connections_count();
-				number_of_peers = 0;
-				m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt)
-				{
-					if(!cntxt.m_is_income)
-						++number_of_peers;
-					return true;
-				}); // lambda
+				if(!cntxt.m_is_income)
+					++number_of_peers;
+				return true;
+			}); // lambda
+				
+			m_current_number_of_out_peers = number_of_peers;
+			epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers);
 				
-				m_number_of_out_peers = number_of_peers;
-				epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers);
-			}
 			std::this_thread::sleep_for(std::chrono::seconds(1));
 		}
       })); // lambda
@@ -724,14 +722,14 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white)
   {
-	if (m_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit
+	if (m_current_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit
 	{
 		return false;
 	}
-	else if (m_number_of_out_peers > m_config.m_net_config.connections_count)
+	else if (m_current_number_of_out_peers > m_config.m_net_config.connections_count)
 	{
 		m_net_server.get_config_object().del_out_connections(1);
-		m_number_of_out_peers --; // atomic variable, update time = 1s
+		m_current_number_of_out_peers --; // atomic variable, update time = 1s
 		return false;
 	}
     LOG_PRINT_L1("Connecting to " << epee::string_tools::get_ip_string_from_int32(na.ip)  << ":"
@@ -1378,25 +1376,14 @@ namespace nodetool
   
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max)
-	{
-	    using namespace std::chrono;
-		auto point = steady_clock::now();
-		auto time_from_epoh = point.time_since_epoch();
-		auto ms = duration_cast< milliseconds >( time_from_epoh ).count();
-		double ms_f = ms;
-		ms_f /= 1000.;
-		
-		std::ofstream limitFile("log/dr-monero/peers_limit.info", std::ios::app);
-		limitFile.precision(7);
+	{		
 		if(max == -1) {
 			m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT;
-			if (m_save_graph)
-				limitFile << static_cast<int>(ms_f) << " " << P2P_DEFAULT_CONNECTIONS_COUNT << std::endl;
+			epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_config.m_net_config.connections_count);
 			return true;
 		}
-		
+		epee::net_utils::data_logger::get_instance().add_data("peers_limit", max);
 		m_config.m_net_config.connections_count = max;
-		limitFile << static_cast<int>(ms_f) << " " << max << std::endl;
 		return true;
 	}
 	
diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp
index 6b2ee698e..7426e6dc7 100644
--- a/src/p2p/network_throttle-detail.cpp
+++ b/src/p2p/network_throttle-detail.cpp
@@ -220,7 +220,7 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s
 	std::ostringstream oss; oss << "["; 	for (auto sample: m_history) oss << sample.m_size << " ";	 oss << "]" << std::ends;
 	std::string history_str = oss.str();
 
-	_info_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)" 
+	_dbg2_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)" 
         << " Speed AVG=" << std::setw(4) <<  ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]"
         <<           " " << std::setw(4) <<  ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]"
 				<<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec "
@@ -312,7 +312,7 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc
 	if (dbg) {
 		std::ostringstream oss; oss << "["; 	for (auto sample: m_history) oss << sample.m_size << " ";	 oss << "]" << std::ends;
 		std::string history_str = oss.str();
-		_info_c( "net/"+m_nameshort+"_c" ,
+		_dbg1_c( "net/"+m_nameshort+"_c" ,
 			(cts.delay > 0 ? "SLEEP" : "")
 			<< "dbg " << m_name << ": " 
 			<< "speed is A=" << std::setw(8) <<cts.average<<" vs "