Merge pull request #2877

43f5269f Wallets now do not depend on the daemon rpc lib (moneromooo-monero)
bb89ae8b move connection_basic and network_throttle from src/p2p to epee (moneromooo-monero)
4abf25f3 cryptonote_core does not depend on p2p anymore (moneromooo-monero)

4 files changed, 444 insertions, 2 deletions

diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h
index 33fec8ec5..c0401c8b0 100644
--- a/contrib/epee/include/net/abstract_tcp_server2.h
+++ b/contrib/epee/include/net/abstract_tcp_server2.h
@@ -54,8 +54,8 @@
 #include <boost/thread/thread.hpp>
 #include "net_utils_base.h"
 #include "syncobj.h"
-#include "../../../../src/p2p/connection_basic.hpp"
-#include "../../../../src/p2p/network_throttle-detail.hpp"
+#include "connection_basic.hpp"
+#include "network_throttle-detail.hpp"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp
new file mode 100644
index 000000000..16de469a7
--- /dev/null
+++ b/contrib/epee/include/net/connection_basic.hpp
@@ -0,0 +1,141 @@
+/// @file
+/// @author rfree (current maintainer in monero.cc project)
+/// @brief base for connection, contains e.g. the ratelimit hooks
+
+// ! This file might contain variable names same as in template class connection<> 
+// ! from files contrib/epee/include/net/abstract_tcp_server2.*
+// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;)
+// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part)
+// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as:
+
+// Copyright (c) 2014-2017, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file  */
+
+#ifndef INCLUDED_p2p_connection_basic_hpp
+#define INCLUDED_p2p_connection_basic_hpp
+
+
+#include <boost/asio.hpp>
+#include <string>
+#include <vector>
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <atomic>
+
+#include <boost/asio.hpp>
+#include <boost/array.hpp>
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/enable_shared_from_this.hpp>
+#include <boost/interprocess/detail/atomic.hpp>
+#include <boost/thread/thread.hpp>
+
+#include <memory>
+
+#include "net/net_utils_base.h"
+#include "syncobj.h"
+
+namespace epee
+{
+namespace net_utils
+{
+
+  /************************************************************************/
+  /*                                                                      */
+  /************************************************************************/
+  /// Represents a single connection from a client.
+
+class connection_basic_pimpl; // PIMPL for this class
+
+  enum t_connection_type { // type of the connection (of this server), e.g. so that we will know how to limit it
+	  e_connection_type_NET = 0, // default (not used?)
+	  e_connection_type_RPC = 1, // the rpc commands  (probably not rate limited, not chunked, etc)
+	  e_connection_type_P2P = 2  // to other p2p node (probably limited)
+  };
+  
+  std::string to_string(t_connection_type type);
+
+class connection_basic { // not-templated base class for rapid developmet of some code parts
+	public:
+		std::unique_ptr< connection_basic_pimpl > mI; // my Implementation
+
+		// moved here from orginal connecton<> - common member variables that do not depend on template in connection<>
+    volatile uint32_t m_want_close_connection;
+    std::atomic<bool> m_was_shutdown;
+    critical_section m_send_que_lock;
+    std::list<std::string> m_send_que;
+    volatile bool m_is_multithreaded;
+    double m_start_time;
+    /// Strand to ensure the connection's handlers are not called concurrently.
+    boost::asio::io_service::strand strand_;
+    /// Socket for the connection.
+    boost::asio::ip::tcp::socket socket_;
+
+		std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/--
+	public:
+		// first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator
+		connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number);
+
+		virtual ~connection_basic() noexcept(false);
+
+		// various handlers to be called from connection class:
+		void do_send_handler_write(const void * ptr , size_t cb);
+		void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part
+
+		void logger_handle_net_write(size_t size); // network data written
+		void logger_handle_net_read(size_t size); // network data read
+
+		void set_start_time();
+
+		// config for rate limit
+		
+		static void set_rate_up_limit(uint64_t limit);
+		static void set_rate_down_limit(uint64_t limit);
+		static uint64_t get_rate_up_limit();
+		static uint64_t get_rate_down_limit();
+
+		// config misc
+		static void set_tos_flag(int tos); // ToS / QoS flag
+		static int get_tos_flag();
+
+		// handlers and sleep
+		void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?)
+		static void save_limit_to_file(int limit); ///< for dr-monero
+		static double get_sleep_time(size_t cb);
+		
+		static void set_save_graph(bool save_graph);
+};
+
+} // nameserver
+} // nameserver
+
+#endif
+
+
diff --git a/contrib/epee/include/net/network_throttle-detail.hpp b/contrib/epee/include/net/network_throttle-detail.hpp
new file mode 100644
index 000000000..dba15a5ed
--- /dev/null
+++ b/contrib/epee/include/net/network_throttle-detail.hpp
@@ -0,0 +1,125 @@
+/// @file
+/// @author rfree (current maintainer in monero.cc project)
+/// @brief implementaion for throttling of connection (count and rate-limit speed etc)
+
+// Copyright (c) 2014-2017, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/* rfree: throttle details, implementing rate limiting */
+
+
+#ifndef INCLUDED_throttle_detail_hpp
+#define INCLUDED_throttle_detail_hpp
+
+#include "network_throttle.hpp"
+
+namespace epee
+{
+namespace net_utils
+{
+
+
+class network_throttle : public i_network_throttle {
+	private:
+		struct packet_info {
+			size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second)
+			packet_info();
+		};
+
+
+		network_speed_bps m_target_speed;
+		size_t m_network_add_cost; // estimated add cost of headers 
+		size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to
+		size_t m_network_max_segment; // recommended max size of 1 TCP transmission
+
+		const size_t m_window_size; // the number of samples to average over
+		network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot()
+		// TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements
+
+		std::vector< packet_info > m_history; // the history of bw usage
+		network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer
+		network_time_seconds m_start_time; // when we were created
+		bool m_any_packet_yet; // did we yet got any packet to count
+
+		std::string m_name; // my name for debug and logs
+		std::string m_nameshort; // my name for debug and logs (used in log file name)
+
+	// each sample is now 1 second
+	public:
+		network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1);
+		virtual ~network_throttle();
+		virtual void set_name(const std::string &name);
+		virtual void set_target_speed( network_speed_kbps target );
+		virtual network_speed_kbps get_target_speed();
+
+		// add information about events:
+		virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs
+		virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc
+
+		virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc)
+
+		virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic
+
+		// time calculations:
+		virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info)
+
+		virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size
+		virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO
+
+		virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported
+		virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const;  ///< ditto, but for given windows time frame
+		virtual double get_current_speed() const;
+
+	private:
+		virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13
+        virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size);
+        virtual void logger_handle_net(const std::string &filename, double time, size_t size);
+};
+
+/***
+ * The complete set of traffic throttle for one typical connection
+*/
+struct network_throttle_bw {
+	public:
+		network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually)
+		network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually)
+		network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually)
+
+	public:
+		network_throttle_bw(const std::string &name1);
+};
+
+
+
+} // namespace net_utils
+} // namespace epee
+
+
+#endif
+
+
diff --git a/contrib/epee/include/net/network_throttle.hpp b/contrib/epee/include/net/network_throttle.hpp
new file mode 100644
index 000000000..464b34726
--- /dev/null
+++ b/contrib/epee/include/net/network_throttle.hpp
@@ -0,0 +1,176 @@
+/// @file
+/// @author rfree (current maintainer in monero.cc project)
+/// @brief interface for throttling of connection (count and rate-limit speed etc)
+
+// Copyright (c) 2014-2017, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/* rfree: throttle basic interface */
+/* rfree: also includes the manager for singeton/global such objects */
+
+
+#ifndef INCLUDED_network_throttle_hpp
+#define INCLUDED_network_throttle_hpp
+
+#include <boost/asio.hpp>
+#include <string>
+#include <vector>
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <atomic>
+
+#include <boost/asio.hpp>
+#include <boost/array.hpp>
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/enable_shared_from_this.hpp>
+#include <boost/interprocess/detail/atomic.hpp>
+#include <boost/thread/thread.hpp>
+
+#include "syncobj.h"
+
+#include "net/net_utils_base.h" 
+#include "misc_log_ex.h" 
+#include <boost/lambda/bind.hpp>
+#include <boost/lambda/lambda.hpp>
+#include <boost/uuid/random_generator.hpp>
+#include <boost/chrono.hpp>
+#include <boost/utility/value_init.hpp>
+#include <boost/asio/deadline_timer.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/thread/thread.hpp> 
+#include "misc_language.h"
+#include "pragma_comp_defs.h"
+#include <sstream>
+#include <iomanip>
+#include <algorithm>
+
+#include <memory>
+#include <mutex>
+#include <fstream>
+
+namespace epee
+{
+namespace net_utils
+{
+
+// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono?
+typedef double network_speed_kbps;   // externally, for parameters and return values, all defined in kilobytes per second
+typedef double network_speed_bps;    // throttle-internally, bytes per second
+typedef double network_time_seconds;
+typedef double network_MB;
+
+class i_network_throttle;
+
+/***
+@brief All information about given throttle - speed calculations
+*/
+struct calculate_times_struct {
+		double average;
+		double window;
+		double delay;
+		double recomendetDataSize;
+};
+typedef calculate_times_struct calculate_times_struct;
+
+
+namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working
+
+/*** 
+@brief Access to simple throttles, with singlton to access global network limits
+*/
+class network_throttle_manager {
+	// provides global (singleton) in/inreq/out throttle access
+
+	// [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/
+	// [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests)
+	
+	//protected:
+	public: // XXX
+
+    static boost::mutex m_lock_get_global_throttle_in;
+    static boost::mutex m_lock_get_global_throttle_inreq;
+    static boost::mutex m_lock_get_global_throttle_out;
+
+		friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS!
+		friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS!
+		friend class connection_basic_pimpl; // ditto
+
+		static int xxx;
+
+	public:
+		static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in
+		static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously
+		static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously
+};
+
+
+
+/***
+@brief interface for the throttle, see the derivated class
+*/
+class i_network_throttle {
+	public:
+		virtual void set_name(const std::string &name)=0;
+		virtual void set_target_speed( network_speed_kbps target )=0;
+		virtual network_speed_kbps get_target_speed()=0;
+
+		virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs
+		virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc
+		virtual void tick() =0; // poke and update timers/history
+		
+		// time calculations:
+		
+		virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate:
+		// Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size 
+
+		// Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now
+
+		virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc
+		virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer
+
+		virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future
+
+		virtual double get_time_seconds() const =0; // a timer
+        virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0;
+
+
+};
+
+
+// ... more in the -advanced.h file
+
+
+} // namespace net_utils
+} // namespace epee
+
+
+#endif
+
+
+