2014 network limit 1.0a +utils +toc -doc -drmonero

commands and options for network limiting
works very well e.g. for 50 KiB/sec up and down
ToS (QoS) flag
peer number limit
TODO some spikes in ingress/download
TODO problems when other up and down limit
added "otshell utils" - simple logging (with colors, text files channels)

13 files changed, 1799 insertions, 127 deletions

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
new file mode 100644
index 000000000..18402a61a
--- /dev/null
+++ b/contrib/CMakeLists.txt
@@ -0,0 +1,3 @@
+
+add_subdirectory(otshell_utils)
+
diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h
index 6c613c5d5..1e6223212 100644
--- a/contrib/epee/include/net/abstract_tcp_server2.h
+++ b/contrib/epee/include/net/abstract_tcp_server2.h
@@ -1,3 +1,9 @@
+/**
+@file
+@author from CrypoNote (see copyright below; Andrey N. Sabelnikov)
+@monero rfree
+@brief the connection templated-class for one peer connection
+*/
 // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
 // All rights reserved.
 // 
@@ -26,7 +32,7 @@
 
 
 
-#ifndef  _ABSTRACT_TCP_SERVER2_H_ 
+#ifndef _ABSTRACT_TCP_SERVER2_H_ 
 #define _ABSTRACT_TCP_SERVER2_H_ 
 
 
@@ -36,6 +42,8 @@
 #include <boost/noncopyable.hpp>
 #include <boost/shared_ptr.hpp>
 #include <atomic>
+#include <map>
+#include <memory>
 
 #include <boost/asio.hpp>
 #include <boost/array.hpp>
@@ -46,7 +54,8 @@
 #include <boost/thread/thread.hpp>
 #include "net_utils_base.h"
 #include "syncobj.h"
-
+#include "../../../../src/p2p/connection_basic.hpp"
+#include "../../../../contrib/otshell_utils/utils.hpp"
 
 #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
 
@@ -61,6 +70,12 @@ namespace net_utils
   protected:
     virtual ~i_connection_filter(){}
   };
+  
+  enum t_server_role { // type of the server, e.g. so that we will know how to limit it
+	  NET = 0, // default (not used? used for misc connections maybe?) TODO
+	  RPC = 1, // the rpc commands 
+	  P2P = 2  // to other p2p node
+  };
 
   /************************************************************************/
   /*                                                                      */
@@ -70,13 +85,18 @@ namespace net_utils
   class connection
     : public boost::enable_shared_from_this<connection<t_protocol_handler> >,
     private boost::noncopyable, 
-    public i_service_endpoint
+    public i_service_endpoint,
+    public connection_basic
   {
   public:
     typedef typename t_protocol_handler::connection_context t_connection_context;
     /// Construct a connection with the given io_service.
-    explicit connection(boost::asio::io_service& io_service,
-      typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter);
+   
+    explicit connection( boost::asio::io_service& io_service,
+			typename t_protocol_handler::config_type& config, 
+			std::atomic<long> &ref_sock_count,  // the ++/-- counter 
+			std::atomic<long> &sock_number, // the only increasing ++ number generator
+			i_connection_filter * &pfilter);
 
     virtual ~connection();
     /// Get the socket associated with the connection.
@@ -90,7 +110,8 @@ namespace net_utils
     void call_back_starter();
   private:
     //----------------- i_service_endpoint ---------------------
-    virtual bool do_send(const void* ptr, size_t cb);
+    virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint)
+    virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data
     virtual bool close();
     virtual bool call_run_once_service_io();
     virtual bool request_callback();
@@ -107,29 +128,24 @@ namespace net_utils
     /// Handle completion of a write operation.
     void handle_write(const boost::system::error_code& e, size_t cb);
 
-    /// Strand to ensure the connection's handlers are not called concurrently.
-    boost::asio::io_service::strand strand_;
-
-    /// Socket for the connection.
-    boost::asio::ip::tcp::socket socket_;
-
     /// Buffer for incoming data.
     boost::array<char, 8192> buffer_;
 
     t_connection_context context;
-    volatile uint32_t m_want_close_connection;
-    std::atomic<bool> m_was_shutdown;
-    critical_section m_send_que_lock;
-    std::list<std::string> m_send_que;
-    volatile uint32_t& m_ref_sockets_count;
     i_connection_filter* &m_pfilter;
-    volatile bool m_is_multithreaded;
 
+	// TODO what do they mean about wait on destructor?? --rfree :
     //this should be the last one, because it could be wait on destructor, while other activities possible on other threads
     t_protocol_handler m_protocol_handler;
     //typename t_protocol_handler::config_type m_dummy_config;
     std::list<boost::shared_ptr<connection<t_protocol_handler> > > m_self_refs; // add_ref/release support
     critical_section m_self_refs_lock;
+    critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk()
+    
+    t_server_role m_connection_type;
+
+	public:
+			void setRPcStation();
   };
 
 
@@ -146,9 +162,11 @@ namespace net_utils
     /// Construct the server to listen on the specified TCP address and port, and
     /// serve up files from the given directory.
     boosted_tcp_server();
-    explicit boosted_tcp_server(boost::asio::io_service& external_io_service);
+    explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_server_role s_type);
     ~boosted_tcp_server();
-
+    
+    std::map<std::string, t_server_role> server_type_map;
+	void create_server_type_map();
     bool init_server(uint32_t port, const std::string address = "0.0.0.0");
     bool init_server(const std::string port,  const std::string& address = "0.0.0.0");
 
@@ -254,22 +272,25 @@ namespace net_utils
     /// Acceptor used to listen for incoming connections.
     boost::asio::ip::tcp::acceptor acceptor_;
 
-    /// The next connection to be accepted.
-    connection_ptr new_connection_;
     std::atomic<bool> m_stop_signal_sent;
     uint32_t m_port;
-    volatile uint32_t m_sockets_count;
+	std::atomic<long> m_sock_count;
+	std::atomic<long> m_sock_number;
     std::string m_address;
-    std::string m_thread_name_prefix;
+    std::string m_thread_name_prefix; //TODO: change to enum server_type, now used
     size_t m_threads_count;
     i_connection_filter* m_pfilter;
     std::vector<boost::shared_ptr<boost::thread> > m_threads;
     boost::thread::id m_main_thread_id;
     critical_section m_threads_lock;
-    volatile uint32_t m_thread_index;
-  };
-}
-}
+    volatile uint32_t m_thread_index; // TODO change to std::atomic
+    t_server_role type;
+
+    /// The next connection to be accepted
+    connection_ptr new_connection_;
+  }; // class <>boosted_tcp_server
+} // namespace
+} // namespace
 
 #include "abstract_tcp_server2.inl"
 
diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl
index db3f9e322..8dff192b1 100644
--- a/contrib/epee/include/net/abstract_tcp_server2.inl
+++ b/contrib/epee/include/net/abstract_tcp_server2.inl
@@ -1,3 +1,9 @@
+/**
+@file
+@author from CrypoNote (see copyright below; Andrey N. Sabelnikov)
+@monero rfree
+@brief the connection templated-class for one peer connection
+*/
 // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
 // All rights reserved.
 // 
@@ -26,7 +32,7 @@
 
 
 
-#include "net_utils_base.h"
+//#include "net_utils_base.h"
 #include <boost/lambda/bind.hpp>
 #include <boost/foreach.hpp>
 #include <boost/lambda/lambda.hpp>
@@ -34,9 +40,20 @@
 #include <boost/chrono.hpp>
 #include <boost/utility/value_init.hpp>
 #include <boost/asio/deadline_timer.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp> // TODO
+#include <boost/thread/thread.hpp> // TODO
 #include "misc_language.h"
 #include "pragma_comp_defs.h"
 
+#include <sstream>
+#include <iomanip>
+#include <algorithm>
+
+#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
+
+#include "../../../../contrib/otshell_utils/utils.hpp"
+using namespace nOT::nUtils; // TODO
+
 PRAGMA_WARNING_PUSH
 namespace epee
 {
@@ -48,17 +65,19 @@ namespace net_utils
 PRAGMA_WARNING_DISABLE_VS(4355)
 
   template<class t_protocol_handler>
-  connection<t_protocol_handler>::connection(boost::asio::io_service& io_service,
-    typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter)
-                          : strand_(io_service),
-                            socket_(io_service),
-                            m_want_close_connection(0), 
-                            m_was_shutdown(0), 
-                            m_ref_sockets_count(sock_count), 
-                            m_pfilter(pfilter),
-                            m_protocol_handler(this, config, context)
-  {
-    boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count);
+  connection<t_protocol_handler>::connection( boost::asio::io_service& io_service,
+  	typename t_protocol_handler::config_type& config, 
+		std::atomic<long> &ref_sock_count,  // the ++/-- counter 
+		std::atomic<long> &sock_number, // the only increasing ++ number generator
+		i_connection_filter* &pfilter
+	)
+	: 
+		connection_basic(io_service, ref_sock_count, sock_number), 
+		m_protocol_handler(this, config, context),
+		m_pfilter( pfilter ),
+		m_connection_type(NET)
+  {
+    _info_c("net/sleepRPC", "connection constructor set m_connection_type="<<m_connection_type);
   }
 PRAGMA_WARNING_DISABLE_VS(4355)
   //---------------------------------------------------------------------------------
@@ -67,12 +86,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   {
     if(!m_was_shutdown)
     {
-      LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown.");
+      _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown.");
       shutdown();
     }
 
-    LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed");
-    boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count);
+    _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed");
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
@@ -118,13 +136,13 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong());
 
     context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income);
-    LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
+    _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
       " to " << local_ep.address().to_string() << ':' << local_ep.port() <<
-      ", total sockets objects " << m_ref_sockets_count);
+      ", total sockets objects " << m_ref_sock_count);
 
     if(m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip))
     {
-      LOG_PRINT_L2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection");
+      _dbg2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection");
       close();
       return false;
     }
@@ -136,7 +154,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         boost::bind(&connection<t_protocol_handler>::handle_read, self,
           boost::asio::placeholders::error,
           boost::asio::placeholders::bytes_transferred)));
-
+          
+	//set ToS flag
+	int tos = get_tos_flag();
+	boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS >
+	optionTos( tos );
+    socket_.set_option( optionTos );
+	//_dbg1("Set ToS flag to " << tos);
+	
     return true;
 
     CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false);
@@ -146,7 +171,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   bool connection<t_protocol_handler>::request_callback()
   {
     TRY_ENTRY();
-    LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback");
+    _dbg2("[" << print_connection_context_short(context) << "] request_callback");
     // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
     auto self = safe_shared_from_this();
     if(!self)
@@ -167,7 +192,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   bool connection<t_protocol_handler>::add_ref()
   {
     TRY_ENTRY();
-    LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref");
+    //_info("[sock " << socket_.native_handle() << "] add_ref");
     CRITICAL_REGION_LOCAL(m_self_refs_lock);
 
     // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@@ -201,7 +226,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   void connection<t_protocol_handler>::call_back_starter()
   {
     TRY_ENTRY();
-    LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback");
+    _dbg2("[" << print_connection_context_short(context) << "] fired_callback");
     m_protocol_handler.handle_qued_callback();
     CATCH_ENTRY_L0("connection<t_protocol_handler>::call_back_starter()", void());
   }
@@ -211,17 +236,18 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     std::size_t bytes_transferred)
   {
     TRY_ENTRY();
-    LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async read calledback.");
+    //_info("[sock " << socket_.native_handle() << "] Async read calledback.");
     
     if (!e)
     {
-      LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4);
+      //_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred);
+      logger_handle_net_read(bytes_transferred);
       context.m_last_recv = time(NULL);
       context.m_recv_cnt += bytes_transferred;
       bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred);
       if(!recv_res)
       {  
-        LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4);
+        //_info("[sock " << socket_.native_handle() << "] protocol_want_close");
 
         //some error in protocol, protocol handler ask to close connection
         boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1);
@@ -239,14 +265,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
             boost::bind(&connection<t_protocol_handler>::handle_read, connection<t_protocol_handler>::shared_from_this(),
               boost::asio::placeholders::error,
               boost::asio::placeholders::bytes_transferred)));
-        LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Async read requested.");
+        //_info("[sock " << socket_.native_handle() << "]Async read requested.");
       }
     }else
     {
-      LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
+      _dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
       if(e.value() != 2)
       {
-        LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
+        _dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
         shutdown();
       }
     }
@@ -283,8 +309,87 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     CATCH_ENTRY_L0("connection<t_protocol_handler>::call_run_once_service_io", false);
   }
   //---------------------------------------------------------------------------------
+    template<class t_protocol_handler>
+  bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) {
+    TRY_ENTRY();
+
+    // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
+    auto self = safe_shared_from_this();
+    if (!self) return false;
+    if (m_was_shutdown) return false;
+		// TODO avoid copy
+
+		const double factor = 32; // TODO config
+		typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic
+		const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) );
+        const t_safe chunksize_max = chunksize_good * 2 ;
+		const bool allow_split = (m_connection_type == RPC) ? false : true; // TODO config
+
+		if (allow_split && (cb > chunksize_max)) {
+			{ // LOCK: chunking
+    		epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** 
+
+				_mark_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
+				_mark("do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
+				t_safe all = cb; // all bytes to send 
+				t_safe pos = 0; // current sending position
+				// 01234567890 
+				// ^^^^        (pos=0, len=4)     ;   pos:=pos+len, pos=4
+				//     ^^^^    (pos=4, len=4)     ;   pos:=pos+len, pos=8
+				//         ^^^ (pos=8, len=4)    ;   
+
+				// const size_t bufsize = chunksize_good; // TODO safecast
+				// char* buf = new char[ bufsize ];
+
+				bool all_ok = true;
+				while (pos < all) {
+					t_safe lenall = all-pos; // length from here to end
+					t_safe len = std::min( chunksize_good , lenall); // take a smaller part
+					ASRT(len<=chunksize_good);
+					// pos=8; len=4; all=10;	len=3;
+
+					ASRT(len>0); ASRT(len < std::numeric_limits<size_t>::max());   // yeap we want strong < then max size, to be sure
+					
+					void *chunk_start = ((char*)ptr) + pos;
+					_fact_c("net/out/size","chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos);
+					ASRT(chunk_start >= ptr); // not wrapped around address?
+					//std::memcpy( (void*)buf, chunk_start, len);
+
+					_dbg1_c("net/out/size", "part of " << lenall << ": pos="<<pos << " len="<<len);
+
+					bool ok = do_send_chunk(chunk_start, len); // <====== ***
+
+					all_ok = all_ok && ok;
+					if (!all_ok) {
+						_mark_c("net/out/size", "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
+						_mark  (                "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
+						_dbg1("do_send() SEND was aborted in middle of big package - this is mostly harmless "
+							<< " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb);
+						return false; // partial failure in sending
+					}
+					pos = pos+len; ASRT(pos >0);
+
+					// (in catch block, or uniq pointer) delete buf;
+				} // each chunk
+
+				_mark_c("net/out/size", "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
+				_mark  (                "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
+
+                _info_c("net/sleepRPC", "do_send() m_connection_type = " << m_connection_type);
+
+				return all_ok; // done - e.g. queued - all the chunks of current do_send call
+			} // LOCK: chunking
+		} // a big block (to be chunked) - all chunks
+		else { // small block
+			return do_send_chunk(ptr,cb); // just send as 1 big chunk
+		}
+
+    CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
+	} // do_send()
+
+  //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
-  bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb)
+  bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb)
   {
     TRY_ENTRY();
     // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@@ -294,49 +399,80 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     if(m_was_shutdown)
       return false;
 
-    LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4);
+    //_info("[sock " << socket_.native_handle() << "] SEND " << cb);
     context.m_last_send = time(NULL);
     context.m_send_cnt += cb;
     //some data should be wrote to stream
     //request complete
     
-    epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock);
-    if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
-    {
-      send_guard.unlock();
-      LOG_PRINT_L2("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
-      close();
-      return false;
+    do_send_handler_start( ptr , cb ); // (((H)))
+
+    epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); // *** critical ***
+    long int retry=0;
+    const long int retry_limit = 5*4;
+    while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
+    {
+        retry++;
+
+        /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop
+            _fact("ABORT queue wait due to stopping");
+            return false; // aborted
+        }*/
+
+        long int ms = 250 + (rand()%50);
+        _info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
+        boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
+        _dbg1("sleep for queue: " << ms);
+
+        if (retry > retry_limit) {
+            send_guard.unlock();
+            _erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
+            //	_mark_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
+            close();
+            return false;
+        }
     }
 
     m_send_que.resize(m_send_que.size()+1);
     m_send_que.back().assign((const char*)ptr, cb);
     
     if(m_send_que.size() > 1)
-    {
-      //active operation should be in progress, nothing to do, just wait last operation callback
-    }else
-    {
-      //no active operation
-      if(m_send_que.size()!=1)
-      {
-        LOG_ERROR("Looks like no active operations, but send que size != 1!!");
-        return false;
-      }
+    { // active operation should be in progress, nothing to do, just wait last operation callback
+        auto size_now = cb;
+        _info_c("net/out/size", "do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
+        do_send_handler_delayed( ptr , size_now ); // (((H)))
 
-      boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()),
-        //strand_.wrap(
-        boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
-        //)
-        );
-      
-      LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
     }
+    else
+    { // no active operation
+
+        if(m_send_que.size()!=1)
+        {
+            _erro("Looks like no active operations, but send que size != 1!!");
+            return false;
+        }
+
+        auto size_now = m_send_que.front().size();
+        _mark_c("net/out/size", "do_send() NOW SENSD: packet="<<size_now<<" B");
+        do_send_handler_write( ptr , size_now ); // (((H)))
+
+        ASRT( size_now == m_send_que.front().size() );
+        boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) ,
+                                 //strand_.wrap(
+                                 boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
+                                 //)
+                                 );
+        //_dbg3("(chunk): " << size_now);
+        logger_handle_net_write(size_now);
+        //_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
+    }
+    
+    do_send_handler_stop( ptr , cb );
 
     return true;
 
     CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
-  }
+  } // do_send_chunk
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
   bool connection<t_protocol_handler>::shutdown()
@@ -353,7 +489,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   bool connection<t_protocol_handler>::close()
   {
     TRY_ENTRY();
-    LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called.");
+    //_info("[sock " << socket_.native_handle() << "] Que Shutdown called.");
     size_t send_que_size = 0;
     CRITICAL_REGION_BEGIN(m_send_que_lock);
     send_que_size = m_send_que.size();
@@ -376,7 +512,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 
     if (e)
     {
-      LOG_PRINT_L1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
+      _dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
       shutdown();
       return;
     }
@@ -385,7 +521,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     CRITICAL_REGION_BEGIN(m_send_que_lock);
     if(m_send_que.empty())
     {
-      LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!");
+      _erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!");
       return;
     }
 
@@ -399,10 +535,17 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     }else
     {
       //have more data to send
-      boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()),
-        //strand_.wrap(
-          boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2));
-        //);
+		auto size_now = m_send_que.front().size();
+		_mark_c("net/out/size", "handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from  queue size="<<m_send_que.size());
+		do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H)))
+		ASRT( size_now == m_send_que.front().size() );
+		boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now) , 
+        // strand_.wrap(
+          boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)
+				// )
+        );
+      //_dbg3("(normal)" << size_now);
+      logger_handle_net_write(size_now);
     }
     CRITICAL_REGION_END();
 
@@ -412,6 +555,13 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     }
     CATCH_ENTRY_L0("connection<t_protocol_handler>::handle_write", void());
   }
+  //---------------------------------------------------------------------------------
+  template<class t_protocol_handler>
+  void connection<t_protocol_handler>::setRPcStation()
+  {
+	m_connection_type = RPC; 
+	_fact_c("net/sleepRPC", "set m_connection_type = RPC ");
+  }
   /************************************************************************/
   /*                                                                      */
   /************************************************************************/
@@ -420,19 +570,27 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     m_io_service_local_instance(new boost::asio::io_service()),
     io_service_(*m_io_service_local_instance.get()),
     acceptor_(io_service_),
-    new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)), 
-    m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0)
+    m_stop_signal_sent(false), m_port(0), 
+	m_sock_count(0), m_sock_number(0), m_threads_count(0), 
+	m_pfilter(NULL), m_thread_index(0),
+    new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter))
   {
+    create_server_type_map();
     m_thread_name_prefix = "NET";
+    type = NET;
   }
 
   template<class t_protocol_handler>
-  boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service):
+  boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_server_role s_type):
     io_service_(extarnal_io_service),
     acceptor_(io_service_),
-    new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)), 
-    m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0)
+    m_stop_signal_sent(false), m_port(0), 
+		m_sock_count(0), m_sock_number(0), m_threads_count(0), 
+		m_pfilter(NULL), m_thread_index(0),
+		type(NET),
+    new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter))
   {
+  create_server_type_map();
     m_thread_name_prefix = "NET";
   }
   //---------------------------------------------------------------------------------
@@ -444,6 +602,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
+  void boosted_tcp_server<t_protocol_handler>::create_server_type_map() 
+  {
+		server_type_map["NET"] = t_server_role::NET;
+		server_type_map["RPC"] = t_server_role::RPC;
+		server_type_map["P2P"] = t_server_role::P2P;
+  }
+  //---------------------------------------------------------------------------------
+  template<class t_protocol_handler>
   bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address)
   {
     TRY_ENTRY();
@@ -491,6 +657,7 @@ POP_WARNINGS
     std::string thread_name = std::string("[") + m_thread_name_prefix;
     thread_name += boost::to_string(local_thr_index) + "]";
     log_space::log_singletone::set_thread_log_prefix(thread_name);
+    //   _fact("Thread name: " << m_thread_name_prefix);
     while(!m_stop_signal_sent)
     {
       try
@@ -499,14 +666,14 @@ POP_WARNINGS
       }
       catch(const std::exception& ex)
       {
-        LOG_ERROR("Exception at server worker thread, what=" << ex.what());
+        _erro("Exception at server worker thread, what=" << ex.what());
       }
       catch(...)
       {
-        LOG_ERROR("Exception at server worker thread, unknown execption");
+        _erro("Exception at server worker thread, unknown execption");
       }
     }
-    LOG_PRINT_L4("Worker thread finished");
+    //_info("Worker thread finished");
     return true;
     CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::worker_thread", false);
   }
@@ -515,6 +682,9 @@ POP_WARNINGS
   void boosted_tcp_server<t_protocol_handler>::set_threads_prefix(const std::string& prefix_name)
   {
     m_thread_name_prefix = prefix_name;
+    type = server_type_map[m_thread_name_prefix];
+    _note("Set server type to: " << type);
+    _note("Set server type to: " << m_thread_name_prefix);
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
@@ -539,32 +709,38 @@ POP_WARNINGS
       {
         boost::shared_ptr<boost::thread> thread(new boost::thread(
           attrs, boost::bind(&boosted_tcp_server<t_protocol_handler>::worker_thread, this)));
+          _note("Run server thread name: " << m_thread_name_prefix);
         m_threads.push_back(thread);
       }
       CRITICAL_REGION_END();
       // Wait for all threads in the pool to exit.
-      if(wait)
+      if (wait)  // && ! ::cryptonote::core::get_is_stopping()) // TODO fast_exit
       {
-        for (std::size_t i = 0; i < m_threads.size(); ++i)
-          m_threads[i]->join();
+		_fact("JOINING all threads");
+        for (std::size_t i = 0; i < m_threads.size(); ++i) {
+			m_threads[i]->join();
+         }
+         _fact("JOINING all threads - almost");
         m_threads.clear();
+        _fact("JOINING all threads - DONE");
 
-      }else
-      {
+      } 
+      else {
+		_dbg1("Reiniting OK.");
         return true;
       }
 
       if(wait && !m_stop_signal_sent)
       {
         //some problems with the listening socket ?..
-        LOG_PRINT_L0("Net service stopped without stop request, restarting...");
+        _dbg1("Net service stopped without stop request, restarting...");
         if(!this->init_server(m_port, m_address))
         {
-          LOG_PRINT_L0("Reiniting service failed, exit.");
+          _dbg1("Reiniting service failed, exit.");
           return false;
         }else
         {
-          LOG_PRINT_L0("Reiniting OK.");
+          _dbg1("Reiniting OK.");
         }
       }
     }
@@ -597,7 +773,7 @@ POP_WARNINGS
     {
       if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms))
       {
-        LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle());
+        _dbg1("Interrupting thread " << m_threads[i]->native_handle());
         m_threads[i]->interrupt();
       }
     }
@@ -626,19 +802,22 @@ POP_WARNINGS
     TRY_ENTRY();
     if (!e)
     {
-      connection_ptr conn(std::move(new_connection_));
-
-      new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter));
+		if (type == RPC) {
+			new_connection_->setRPcStation();
+			_note("New server for RPC connections");
+		}
+		connection_ptr conn(std::move(new_connection_));
+      new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter));
       acceptor_.async_accept(new_connection_->socket(),
         boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this,
         boost::asio::placeholders::error));
 
       bool r = conn->start(true, 1 < m_threads_count);
       if (!r)
-        LOG_ERROR("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count);
+        _erro("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
     }else
     {
-      LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count);
+      _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count);
     }
     CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::handle_accept", void());
   }
@@ -648,7 +827,7 @@ POP_WARNINGS
   {
     TRY_ENTRY();
 
-    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) );
+    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) );
     boost::asio::ip::tcp::socket&  sock_ = new_connection_l->socket();
     
     //////////////////////////////////////////////////////////////////////////
@@ -658,7 +837,7 @@ POP_WARNINGS
     boost::asio::ip::tcp::resolver::iterator end;
     if(iterator == end)
     {
-      LOG_ERROR("Failed to resolve " << adr);
+      _erro("Failed to resolve " << adr);
       return false;
     }
     //////////////////////////////////////////////////////////////////////////
@@ -704,7 +883,7 @@ POP_WARNINGS
       {
         //timeout
         sock_.close();
-        LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
+        _dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
         return false;
       }
     }
@@ -712,21 +891,21 @@ POP_WARNINGS
 
     if (ec || !sock_.is_open())
     {
-      LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3);
+      _dbg3("Some problems at connect, message: " << ec.message());
       return false;
     }
 
-    LOG_PRINT_L3("Connected success to " << adr << ':' << port);
+    _dbg3("Connected success to " << adr << ':' << port);
 
     bool r = new_connection_l->start(false, 1 < m_threads_count);
     if (r)
     {
       new_connection_l->get_context(conn_context);
-      //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter));
+      //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_pfilter));
     }
     else
     {
-      LOG_ERROR("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count);
+      _erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
     }
 
     return r;
@@ -738,7 +917,7 @@ POP_WARNINGS
   bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip)
   {
     TRY_ENTRY();    
-    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) );
+    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) );
     boost::asio::ip::tcp::socket&  sock_ = new_connection_l->socket();
     
     //////////////////////////////////////////////////////////////////////////
@@ -748,7 +927,7 @@ POP_WARNINGS
     boost::asio::ip::tcp::resolver::iterator end;
     if(iterator == end)
     {
-      LOG_ERROR("Failed to resolve " << adr);
+      _erro("Failed to resolve " << adr);
       return false;
     }
     //////////////////////////////////////////////////////////////////////////
@@ -768,7 +947,7 @@ POP_WARNINGS
       {
           if(error != boost::asio::error::operation_aborted) 
           {
-            LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
+            _dbg3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
             new_connection_l->socket().close();
           }
       });
@@ -785,7 +964,7 @@ POP_WARNINGS
             cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation
           }else
           {
-            LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
+            _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
               " from " << lep.address().to_string() << ':' << lep.port());
             bool r = new_connection_l->start(false, 1 < m_threads_count);
             if (r)
@@ -795,13 +974,13 @@ POP_WARNINGS
             }
             else
             {
-              LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
+              _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
               cb(conn_context, boost::asio::error::fault);
             }
           }
         }else
         {
-          LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
+          _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
             " from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value());
           cb(conn_context, ec_);
         }
@@ -809,6 +988,6 @@ POP_WARNINGS
     return true;
     CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::connect_async", false);
   }
-}
-}
+} // namespace
+} // namespace
 PRAGMA_WARNING_POP
diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h
index e79768b04..5e5e803f5 100644
--- a/contrib/epee/include/net/levin_protocol_handler_async.h
+++ b/contrib/epee/include/net/levin_protocol_handler_async.h
@@ -81,6 +81,7 @@ public:
 
   async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE)
   {}
+  void del_connections(size_t count);
 };
 
 
@@ -669,6 +670,14 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context>
+void async_protocol_handler_config<t_connection_context>::del_connections(size_t count) // TODO
+{
+	CRITICAL_REGION_BEGIN(m_connects_lock);
+	m_connects.clear();		
+	CRITICAL_REGION_END();
+}
+//------------------------------------------------------------------------------------------
+template<class t_connection_context>
 void async_protocol_handler_config<t_connection_context>::add_connection(async_protocol_handler<t_connection_context>* pconn)
 {
   CRITICAL_REGION_BEGIN(m_connects_lock);
diff --git a/contrib/otshell_utils/CMakeLists.txt b/contrib/otshell_utils/CMakeLists.txt
new file mode 100644
index 000000000..7413e0dc5
--- /dev/null
+++ b/contrib/otshell_utils/CMakeLists.txt
@@ -0,0 +1,14 @@
+cmake_minimum_required (VERSION 2.6)
+project (otshell CXX)
+
+# Add executable
+
+file(GLOB otshell_utils_sources # All files in directory:
+	"*.h"
+	"*.hpp"
+	"*.cpp"
+)
+
+add_library (otshell_utils STATIC ${otshell_utils_sources})
+set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils")
+#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs
diff --git a/contrib/otshell_utils/LICENCE.txt b/contrib/otshell_utils/LICENCE.txt
new file mode 100644
index 000000000..f351acf10
--- /dev/null
+++ b/contrib/otshell_utils/LICENCE.txt
@@ -0,0 +1,21 @@
+
+This are some files also from OpenTransactions / otshell project,
+developed thanks to the awesome OpenTransaction project, organization and developers :)
+
+Parts of code here was also developed thanks to the excellent Monero project,
+thanks to Monero project, organization and developers :)
+
+[Some] files/code here (in external/otshell_utils) are under licence defined in
+src/doc/LICENCE-otshell.txt ;
+Others are from monero, with licence in src/doc/LICENCE-monero.txt ;
+
+For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code,
+possibly not all) I do not worry who uses it how; I'am not a lawyer.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project
+Monero and Open Transactions, to share and develop both faster.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+
diff --git a/contrib/otshell_utils/ccolor.cpp b/contrib/otshell_utils/ccolor.cpp
new file mode 100644
index 000000000..cd93e0de7
--- /dev/null
+++ b/contrib/otshell_utils/ccolor.cpp
@@ -0,0 +1,116 @@
+#include "ccolor.hpp"
+#include <cstdarg>
+
+// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
+// from http://wiznet.gr/src/ccolor.zip
+// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
+
+using namespace std;
+
+#ifdef _MSC_VER
+
+#define snprintf c99_snprintf
+
+inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) {
+	int count = -1;
+	if (size != 0)
+		 count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
+	if (count == -1)
+		 count = _vscprintf(format, ap);
+	return count;
+}
+
+inline int c99_snprintf(char* str, size_t size, const char* format, ...) {
+	int count;
+	va_list ap;
+	va_start(ap, format);
+	count = c99_vsnprintf(str, size, format, ap);
+	va_end(ap);
+	return count;
+}
+#endif // _MSC_VER
+
+#define CC_CONSOLE_COLOR_DEFAULT "\033[0m"
+#define CC_FORECOLOR(C) "\033[" #C "m"
+#define CC_BACKCOLOR(C) "\033[" #C "m"
+#define CC_ATTR(A) "\033[" #A "m"
+
+namespace zkr
+{
+	enum Color
+	{
+		Black,
+		Red,
+		Green,
+		Yellow,
+		Blue,
+		Magenta,
+		Cyan,
+		White,
+		Default = 9
+	};
+
+	enum Attributes
+	{
+		Reset,
+		Bright,
+		Dim,
+		Underline,
+		Blink,
+		Reverse,
+		Hidden
+	};
+
+	char * cc::color(int attr, int fg, int bg)
+	{
+				static const int size = 20;
+        static char command[size];
+
+        /* Command is the control command to the terminal */
+        snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40);
+        return command;
+	}
+
+
+	const char *cc::console = CC_CONSOLE_COLOR_DEFAULT;
+	const char *cc::underline = CC_ATTR(4);
+	const char *cc::bold = CC_ATTR(1);
+
+	const char *cc::fore::black = CC_FORECOLOR(30);
+	const char *cc::fore::blue = CC_FORECOLOR(34);
+	const char *cc::fore::red = CC_FORECOLOR(31);
+	const char *cc::fore::magenta = CC_FORECOLOR(35);
+	const char *cc::fore::green = CC_FORECOLOR(92);
+	const char *cc::fore::cyan = CC_FORECOLOR(36);
+	const char *cc::fore::yellow = CC_FORECOLOR(33);
+	const char *cc::fore::white = CC_FORECOLOR(37);
+	const char *cc::fore::console = CC_FORECOLOR(39);
+
+	const char *cc::fore::lightblack = CC_FORECOLOR(90);
+	const char *cc::fore::lightblue = CC_FORECOLOR(94);
+	const char *cc::fore::lightred = CC_FORECOLOR(91);
+	const char *cc::fore::lightmagenta = CC_FORECOLOR(95);
+	const char *cc::fore::lightgreen = CC_FORECOLOR(92);
+	const char *cc::fore::lightcyan = CC_FORECOLOR(96);
+	const char *cc::fore::lightyellow = CC_FORECOLOR(93);
+	const char *cc::fore::lightwhite = CC_FORECOLOR(97);
+
+	const char *cc::back::black = CC_BACKCOLOR(40);
+	const char *cc::back::blue = CC_BACKCOLOR(44);
+	const char *cc::back::red = CC_BACKCOLOR(41);
+	const char *cc::back::magenta = CC_BACKCOLOR(45);
+	const char *cc::back::green = CC_BACKCOLOR(42);
+	const char *cc::back::cyan = CC_BACKCOLOR(46);
+	const char *cc::back::yellow = CC_BACKCOLOR(43);
+	const char *cc::back::white = CC_BACKCOLOR(47);
+	const char *cc::back::console = CC_BACKCOLOR(49);
+
+	const char *cc::back::lightblack = CC_BACKCOLOR(100);
+	const char *cc::back::lightblue = CC_BACKCOLOR(104);
+	const char *cc::back::lightred = CC_BACKCOLOR(101);
+	const char *cc::back::lightmagenta = CC_BACKCOLOR(105);
+	const char *cc::back::lightgreen = CC_BACKCOLOR(102);
+	const char *cc::back::lightcyan = CC_BACKCOLOR(106);
+	const char *cc::back::lightyellow = CC_BACKCOLOR(103);
+	const char *cc::back::lightwhite = CC_BACKCOLOR(107);
+}
diff --git a/contrib/otshell_utils/ccolor.hpp b/contrib/otshell_utils/ccolor.hpp
new file mode 100644
index 000000000..bf5a601a2
--- /dev/null
+++ b/contrib/otshell_utils/ccolor.hpp
@@ -0,0 +1,73 @@
+// ccolor.hpp
+
+// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
+// from http://wiznet.gr/src/ccolor.zip
+// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
+
+#ifndef INCLUDE_OT_ccolor
+#define INCLUDE_OT_ccolor
+
+#include <iostream>
+#include <stdio.h>
+
+namespace zkr
+{
+	class cc
+	{
+	public:
+
+		class fore
+		{
+		public:
+			static const char *black;
+			static const char *blue;
+			static const char *red;
+			static const char *magenta;
+			static const char *green;
+			static const char *cyan;
+			static const char *yellow;
+			static const char *white;
+			static const char *console;
+
+			static const char *lightblack;
+			static const char *lightblue;
+			static const char *lightred;
+			static const char *lightmagenta;
+			static const char *lightgreen;
+			static const char *lightcyan;
+			static const char *lightyellow;
+			static const char *lightwhite;
+		};
+
+		class back
+		{
+		public:
+			static const char *black;
+			static const char *blue;
+			static const char *red;
+			static const char *magenta;
+			static const char *green;
+			static const char *cyan;
+			static const char *yellow;
+			static const char *white;
+			static const char *console;
+
+			static const char *lightblack;
+			static const char *lightblue;
+			static const char *lightred;
+			static const char *lightmagenta;
+			static const char *lightgreen;
+			static const char *lightcyan;
+			static const char *lightyellow;
+			static const char *lightwhite;
+		};
+
+		static char *color(int attr, int fg, int bg);
+		static const char *console;
+		static const char *underline;
+		static const char *bold;
+	};
+}
+
+#endif
+
diff --git a/contrib/otshell_utils/lib_common1.hpp b/contrib/otshell_utils/lib_common1.hpp
new file mode 100644
index 000000000..108b1847c
--- /dev/null
+++ b/contrib/otshell_utils/lib_common1.hpp
@@ -0,0 +1,51 @@
+/* See other files here for the LICENCE that applies here. */
+
+
+#ifndef INCLUDE_OT_NEWCLI_COMMON1
+#define INCLUDE_OT_NEWCLI_COMMON1
+
+#include <string>
+#include <cstring>
+#include <vector>
+#include <map>
+#include <list>
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <set>
+#include <iterator>
+#include <stdexcept>
+
+#include <functional>
+#include <memory>
+#include <thread>
+#include <mutex>
+
+
+// list of thigs from libraries that we pull into namespace nOT::nNewcli
+// we might still need to copy/paste it in few places to make IDEs pick it up correctly
+#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \
+	using std::string; \
+	using std::vector; \
+	using std::vector; \
+	using std::list; \
+	using std::set; \
+	using std::map; \
+	using std::ostream; \
+	using std::istream; \
+	using std::cin; \
+	using std::cerr; \
+	using std::cout; \
+	using std::cerr; \
+	using std::endl; \
+	using std::function; \
+	using std::unique_ptr; \
+	using std::shared_ptr; \
+	using std::weak_ptr; \
+	using std::enable_shared_from_this; \
+	using std::mutex; \
+	using std::lock_guard; \
+
+#endif
+
diff --git a/contrib/otshell_utils/runoptions.cpp b/contrib/otshell_utils/runoptions.cpp
new file mode 100644
index 000000000..28e7ceb58
--- /dev/null
+++ b/contrib/otshell_utils/runoptions.cpp
@@ -0,0 +1,69 @@
+/* See other files here for the LICENCE that applies here. */
+/* See header file .hpp for info */
+
+#include "runoptions.hpp"
+
+#include "lib_common1.hpp"
+
+namespace nOT {
+
+INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
+
+// (no debug - this is the default)
+// +nodebug (no debug)
+// +debug ...... --asdf
+// +debug +debugcerr .... --asfs
+// +debug +debugfile .... --asfs
+
+cRunOptions::cRunOptions()
+	: mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false)
+	,mDoRunDebugshow(false)
+{ }
+
+vector<string> cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector<string> & args) {
+	vector<string> arg_clear; // will store only the arguments that are not removed
+
+	for (auto arg : args) {
+		bool thisIsRunoption=false;
+
+		if (arg.size()>0) {
+			if (arg.at(0) == '+') thisIsRunoption=true;
+		}
+
+		if (thisIsRunoption) Exec(arg); // ***
+		if (! thisIsRunoption) arg_clear.push_back(arg);
+	}
+
+	Normalize();
+
+	return arg_clear;
+}
+
+void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug");
+	if (runoption == "+nodebug") { mDebug=false; }
+	else if (runoption == "+debug") { mDebug=true; }
+	else if (runoption == "+debugcerr") { mDebug=true;  mDebugSendToCerr=true; }
+	else if (runoption == "+debugfile") { mDebug=true;  mDebugSendToFile=true; }
+	else if (runoption == "+demo") { mRunMode=eRunModeDemo; }
+	else if (runoption == "+normal") { mRunMode=eRunModeNormal; }
+	else if (runoption == "+current") { mRunMode=eRunModeCurrent; }
+	else if (runoption == "+debugshow") { mDebug=true;  mDebugSendToCerr=true;  mDoRunDebugshow=true; }
+	else {
+		cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl;
+		throw std::runtime_error("Unknown runoption");
+	}
+	// cerr<<"debug="<<mDebug<<endl;
+}
+
+void cRunOptions::Normalize() {
+	if (mDebug) {
+		if (!(  mDebugSendToFile || mDebugSendToCerr  ))  mDebugSendToCerr=true; // if debug is on then send to something, e.g. to cerr
+	}
+}
+
+
+cRunOptions gRunOptions; // (extern)
+
+}; // namespace OT
+
+
diff --git a/contrib/otshell_utils/runoptions.hpp b/contrib/otshell_utils/runoptions.hpp
new file mode 100644
index 000000000..f3306283a
--- /dev/null
+++ b/contrib/otshell_utils/runoptions.hpp
@@ -0,0 +1,58 @@
+/* See other files here for the LICENCE that applies here. */
+/*
+Template for new files, replace word "template" and later delete this line here.
+*/
+
+#ifndef INCLUDE_OT_NEWCLI_runoptions_hpp
+#define INCLUDE_OT_NEWCLI_runoptions_hpp
+
+#include "lib_common1.hpp"
+
+namespace nOT {
+
+INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
+
+/** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug.
+This is NOT for all the other options that are parsed and executed by program. */
+class cRunOptions {
+	public:
+		enum tRunMode { ///< Type of run mode - is this normal, or demonstration etc.
+			eRunModeCurrent=1, ///< currently developed version
+			eRunModeDemo, ///< best currently available Demo of something nice
+			eRunModeNormal, ///< do the normal things that the program should do
+		};
+
+	private:
+		tRunMode mRunMode; ///< selected run mode
+
+		bool mDebug; // turn debug on, Eg: +debug without it probably nothing will be written to debug (maybe just error etc)
+		bool mDebugSendToFile; // send to file, Eg: for +debugfile ; also turns on debug
+		bool mDebugSendToCerr; // send to cerr, Eg: for +debugcerr ; also turns on debug
+		// if debug is set but not any other DebugSend* then we will default to sending to debugcerr
+
+		bool mDoRunDebugshow;
+
+	public:
+		tRunMode getTRunMode() const { return mRunMode; }
+		bool getDebug() const { return mDebug; }
+		bool getDebugSendToFile() const { return mDebugSendToFile; }
+		bool getDebugSendToCerr() const { return mDebugSendToCerr; }
+		bool getDoRunDebugshow() const { return mDoRunDebugshow; }
+
+		cRunOptions();
+
+		vector<string> ExecuteRunoptionsAndRemoveThem(const vector<string> & args);
+		void Exec(const string & runoption); // eg: Exec("+debug");
+
+		void Normalize();
+};
+
+extern cRunOptions gRunOptions;
+
+
+}; // namespace nOT
+
+
+
+#endif
+
diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp
new file mode 100644
index 000000000..489fb3076
--- /dev/null
+++ b/contrib/otshell_utils/utils.cpp
@@ -0,0 +1,612 @@
+/// @file
+/// @author rfree (current maintainer in monero.cc project)
+/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
+
+/* See other files here for the LICENCE that applies here. */
+/* See header file .hpp for info */
+
+#include <algorithm>
+#include <functional>
+#include <cctype>
+#include <locale>
+#include <fstream>
+#include <iostream>
+#include <iomanip>
+
+#include "utils.hpp"
+
+#include "ccolor.hpp"
+
+#include "lib_common1.hpp"
+
+#include "runoptions.hpp"
+
+#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64)
+	#define OS_TYPE_WINDOWS
+#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__)
+	#define OS_TYPE_POSIX
+#else
+	#warning "Compiler/OS platform is not recognized"
+	#warning "Just assuming it will work as POSIX then"
+	#define OS_TYPE_POSIX
+#endif
+
+#if defined(OS_TYPE_WINDOWS)
+	#include <windows.h>
+#elif defined(OS_TYPE_POSIX)
+	#include <sys/types.h>
+	#include <sys/stat.h>
+	#include <unistd.h>
+#else
+	#error "Compiler/OS platform detection failed - not supported"
+#endif
+
+namespace nOT {
+namespace nUtils {
+
+INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
+
+myexception::myexception(const char * what)
+	: std::runtime_error(what)
+{ }
+
+myexception::myexception(const std::string &what)
+	: std::runtime_error(what)
+{ }
+
+void myexception::Report() const {
+	_erro("Error: " << what());
+}
+
+//myexception::~myexception() { }
+
+// ====================================================================
+
+// text trimming
+// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
+std::string & ltrim(std::string &s) {
+	s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace))));
+	return s;
+}
+
+std::string & rtrim(std::string &s) {
+	s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
+  return s;
+}
+
+std::string & trim(std::string &s) {
+	return ltrim(rtrim(s));
+}
+
+std::string get_current_time()
+{
+	std::stringstream stream;
+	struct tm * date;
+
+	std::chrono::high_resolution_clock::time_point now = std::chrono::high_resolution_clock::now();
+	time_t time_now;
+	time_now = std::chrono::high_resolution_clock::to_time_t(now);
+	date = std::localtime(& time_now);
+
+	char date_buff[32];
+	std::strftime(date_buff, sizeof(date_buff), "%d-%b-%Y %H:%M:%S.", date);
+	stream << date_buff;
+
+	std::chrono::high_resolution_clock::duration duration = now.time_since_epoch();
+	int64_t micro = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
+	micro %= 1000000;
+	stream << std::setfill('0') << std::setw(3) << micro;
+
+	return stream.str();
+}
+
+cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data)
+
+std::mutex gLoggerGuard; // extern
+
+// ====================================================================
+
+namespace nDetail {
+
+const char* DbgShortenCodeFileName(const char *s) {
+	const char *p = s;
+	const char *a = s;
+
+	bool inc=1;
+	while (*p) {
+		++p;
+		if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash
+		if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character)
+	}
+	return a;
+}
+
+}
+
+// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795
+#ifndef _MSC_VER
+template<typename T, typename ...Args>
+std::unique_ptr<T> make_unique( Args&& ...args )
+{
+	return std::unique_ptr<T>( new T( std::forward<Args>(args)... ) );
+}
+#else
+	using std::make_unique;
+#endif
+// ====================================================================
+
+char cFilesystemUtils::GetDirSeparator() {
+	// TODO nicer os detection?
+	#if defined(OS_TYPE_POSIX)
+		return '/';
+	#elif defined(OS_TYPE_WINDOWS)
+		return '\\';
+	#else
+		#error "Do not know how to compile this for your platform."
+	#endif
+}
+
+bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) {
+	const bool dbg=false;
+	//struct stat st;
+	const char dirch = cFilesystemUtils::GetDirSeparator();
+	std::istringstream iss(dir);
+	string part, sofar="";
+	if (dir.size()<1) return false; // illegal name
+	// dir[0] is valid from here
+	if  (only_below && (dir[0]==dirch)) return false; // no jumping to top (on any os)
+	while (getline(iss,part,dirch)) {
+		if (dbg) cout << '['<<part<<']' << endl;
+		sofar += part;
+		if (part.size()<1) return false; // bad format?
+		if ((only_below) && (part=="..")) return false; // going up
+
+		if (dbg) cout << "test ["<<sofar<<"]"<<endl;
+		// TODO nicer os detection?
+		#if defined(OS_TYPE_POSIX)
+			struct stat st;
+			bool exists = stat(sofar.c_str() ,&st) == 0; // *
+			if (exists) {
+				if (! S_ISDIR(st.st_mode)) {
+					// std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl;
+					return false; // exists but is a file nor dir
+				}
+			}
+		#elif defined(OS_TYPE_WINDOWS)
+			DWORD dwAttrib = GetFileAttributesA(sofar.c_str());
+			bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY));
+		#else
+			#error "Do not know how to compile this for your platform."
+		#endif
+
+		if (!exists) {
+			if (dbg) cout << "mkdir ["<<sofar<<"]"<<endl;
+			#if defined(OS_TYPE_POSIX)
+				bool ok = 0==  mkdir(sofar.c_str(), 0700); // ***
+			#elif defined(OS_TYPE_WINDOWS)
+				bool ok = (bool) CreateDirectoryA(sofar.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16
+			#else
+				#error "Do not know how to compile this for your platform."
+			#endif
+			if (!ok) return false;
+		}
+		sofar += cFilesystemUtils::GetDirSeparator();
+	}
+	return true;
+}
+// ====================================================================
+
+namespace nDetail {
+
+cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) {
+}
+
+cDebugScopeGuard::~cDebugScopeGuard() {
+	if (mLevel != -1) {
+		gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush;
+	}
+}
+
+void cDebugScopeGuard::Assign(const string &chan, const int level, const string &msg) {
+	mChan=chan;
+	mLevel=level;
+	mMsg=msg;
+}
+
+}; // namespace nDetail
+
+// ====================================================================
+
+cLogger::cLogger() :
+mStream(NULL),
+mLevel(85),
+mThread2Number_Biggest(0) // the CURRENT biggest value (no thread yet in map)
+{
+	mStream = & std::cout;
+	Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1
+}
+
+cLogger::~cLogger() {
+	for (auto pair : mChannels) {
+		std::ofstream *ptr = pair.second;
+		delete ptr;
+		pair.second=NULL;
+	}
+}
+
+std::ostream & cLogger::write_stream(int level) {
+	return write_stream(level,"");
+}
+
+std::ostream & cLogger::write_stream(int level, const std::string & channel ) {
+	if ((level >= mLevel) && (mStream)) {
+		ostream & output = SelectOutput(level,channel);
+		output << icon(level) << ' ';
+		std::thread::id this_id = std::this_thread::get_id();
+		output << "{" << Thread2Number(this_id) << "} ";
+		return output;
+	}
+	return g_nullstream;
+}
+
+std::string cLogger::GetLogBaseDir() const {
+	return "log";
+}
+
+void cLogger::OpenNewChannel(const std::string & channel) {
+	size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparator());
+	// log/test/aaa
+	//         ^----- last_split
+	string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparator() + channel.substr(0, last_split);
+	string basefile = channel.substr(last_split+1) + ".log";
+	string fname = dir + cFilesystemUtils::GetDirSeparator() + cFilesystemUtils::GetDirSeparator() + basefile;
+	_dbg1("Starting debug to channel file: " + fname + " in directory ["+dir+"]");
+	bool dirok = cFilesystemUtils::CreateDirTree(dir);
+	if (!dirok) { const string msg = "In logger failed to open directory (" + dir +")."; _erro(msg); throw std::runtime_error(msg); }
+	std::ofstream * thefile = new std::ofstream( fname.c_str() );
+	*thefile << "====== (Log opened: " << fname << ") ======" << endl;
+	mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) );
+}
+
+std::ostream & cLogger::SelectOutput(int level, const std::string & channel) {
+	if (channel=="") return *mStream;
+	auto obj = mChannels.find(channel);
+	if (obj == mChannels.end()) { // new channel
+		OpenNewChannel(channel);
+		return SelectOutput(level,channel);
+	}
+	else { // existing
+		return * obj->second;
+	}
+}
+
+
+void cLogger::setOutStreamFile(const string &fname) { // switch to using this file
+	_mark("WILL SWITCH DEBUG NOW to file: " << fname);
+	mOutfile =  make_unique<std::ofstream>(fname);
+	mStream = & (*mOutfile);
+	_mark("Started new debug, to file: " << fname);
+}
+
+void cLogger::setOutStreamFromGlobalOptions() {
+	if ( gRunOptions.getDebug() ) {
+		if ( gRunOptions.getDebugSendToFile() ) {
+			mOutfile =  make_unique<std::ofstream> ("debuglog.txt");
+			mStream = & (*mOutfile);
+		}
+		else if ( gRunOptions.getDebugSendToCerr() ) {
+			mStream = & std::cerr;
+		}
+		else {
+			mStream = & g_nullstream;
+		}
+	}
+	else {
+		mStream = & g_nullstream;
+	}
+}
+
+void cLogger::setDebugLevel(int level) {
+	bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level)
+	if (note_before) _note("Setting debug level to "<<level);
+	mLevel = level;
+	if (!note_before) _note("Setting debug level to "<<level);
+}
+
+std::string cLogger::icon(int level) const {
+	// TODO replan to avoid needles converting back and forth char*, string etc
+
+	using namespace zkr;
+
+	if (level >= 100) return cc::back::red     + ToStr(cc::fore::black) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ;
+	if (level >=  90) return cc::back::lightyellow  + ToStr(cc::fore::black) + ToStr("Warn  ") + ToStr(cc::fore::red)+ " " ;
+	if (level >=  80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK  "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " ";
+	if (level >=  75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " ";
+	if (level >=  70) return cc::fore::green    + ToStr("Note  ");
+	if (level >=  50) return cc::fore::cyan   + ToStr("info  ");
+	if (level >=  40) return cc::fore::lightwhite    + ToStr("dbg   ");
+	if (level >=  30) return cc::fore::lightblue   + ToStr("dbg   ");
+	if (level >=  20) return cc::fore::blue    + ToStr("dbg   ");
+
+	return "  ";
+}
+
+std::string cLogger::endline() const {
+	return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc
+}
+
+int cLogger::Thread2Number(const std::thread::id id) {
+	auto found = mThread2Number.find( id );
+	if (found == mThread2Number.end()) { // new one
+		mThread2Number_Biggest++;
+		mThread2Number[id] = mThread2Number_Biggest;
+		return mThread2Number_Biggest;
+	//	_info("(This is a new thread)"); // recursion!
+	} else {
+		return mThread2Number[id];
+	}
+}
+
+
+// ====================================================================
+// object gCurrentLogger is defined later - in global namespace below
+
+
+// ====================================================================
+// vector debug
+
+void DisplayStringEndl(std::ostream & out, const std::string text) {
+	out << text;
+	out << std::endl;
+}
+
+std::string SpaceFromEscape(const std::string &s) {
+	std::ostringstream  newStr;
+		for(size_t i = 0; i < s.length();i++) {
+			if(s[i] == '\\' && s[i+1] ==32)
+				newStr<<"";
+			else
+				newStr<<s[i];
+			}
+	return newStr.str();
+}
+
+std::string EscapeFromSpace(const std::string &s) {
+	std::ostringstream  newStr;
+	for(size_t i = 0; i < s.length();i++) {
+		if(s[i] == 32)
+			newStr << "\\" << " ";
+		else
+			newStr << s[i];
+	}
+	return newStr.str();
+}
+
+
+std::string EscapeString(const std::string &s) {
+	std::ostringstream  newStr;
+		for(size_t i = 0; i < s.length();i++) {
+			if(s[i] >=32 && s[i] <= 126)
+				newStr<<s[i];
+			else
+				newStr<<"\\"<< (int) s[i];
+			}
+
+	return newStr.str();
+}
+
+
+bool CheckIfBegins(const std::string & beggining, const std::string & all) {
+	if (all.compare(0, beggining.length(), beggining) == 0) {
+		return 1;
+	}
+	else {
+		return 0;
+	}
+}
+
+bool CheckIfEnds (std::string const & ending, std::string const & all){
+	if (all.length() >= ending.length()) {
+		return (0 == all.compare (all.length() - ending.length(), ending.length(), ending));
+	} else {
+		return false;
+	}
+}
+
+
+vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib) {
+	vector<string> ret;
+	for ( auto rec : possib) { // check of possibilities
+		if (CheckIfBegins(sofar,rec)) {
+			rec = EscapeFromSpace(rec);
+			ret.push_back(rec); // this record matches
+		}
+	}
+	return ret;
+}
+
+char GetLastChar(const std::string & str) { // TODO unicode?
+	auto s = str.length();
+	if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP);
+	return str.at( s - 1);
+}
+
+std::string GetLastCharIf(const std::string & str) { // TODO unicode?
+	auto s = str.length();
+	if (s==0) return ""; // empty string signalizes ther is nothing to be returned
+	return std::string( 1 , str.at( s - 1) );
+}
+
+// ====================================================================
+
+// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
+// Use it like this: ASRT( x>y );  with the semicolon at end, a clever trick forces this syntax :)
+
+void Assert(bool result, const std::string &stamp, const std::string &condition) {
+	if (!result) {
+		_erro("Assert failed at "+stamp+": ASSERT( " << condition << ")");
+		throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")");
+	}
+}
+
+// ====================================================================
+// advanced string
+
+const std::string GetMultiline(string endLine) {
+	std::string result(""); // Taken from OT_CLI_ReadUntilEOF
+	while (true) {
+		std::string input_line("");
+		if (std::getline(std::cin, input_line, '\n'))
+		{
+			input_line += "\n";
+				if (input_line[0] == '~')
+					break;
+			result += input_line;
+		}
+		if (std::cin.eof() )
+		{
+			std::cin.clear();
+				break;
+		}
+		if (std::cin.fail() )
+		{
+			std::cin.clear();
+				break;
+		}
+		if (std::cin.bad())
+		{
+			std::cin.clear();
+				break;
+		}
+	}
+	return result;
+}
+
+vector<string> SplitString(const string & str){
+		std::istringstream iss(str);
+		vector<string> vec { std::istream_iterator<string>{iss}, std::istream_iterator<string>{} };
+		return vec;
+}
+
+bool checkPrefix(const string & str, char prefix) {
+	if (str.at(0) == prefix)
+		return true;
+	return false;
+}
+
+// ====================================================================
+// operation on files
+
+
+#ifdef __unix
+
+void cEnvUtils::GetTmpTextFile() {
+	// TODO make this name configurable (depending on project)
+	char filename[] = "/tmp/otshellutils_text.XXXXXX";
+	fd = mkstemp(filename);
+	if (fd == -1) {
+		_erro("Can't create the file: " << filename);
+		return;
+	}
+	mFilename = filename;
+}
+
+void cEnvUtils::CloseFile() {
+	close(fd);
+	unlink( mFilename.c_str() );
+}
+
+void  cEnvUtils::OpenEditor() {
+	char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file
+	if (editor == NULL)
+		editor = std::getenv("VISUAL");
+	if (editor == NULL)
+		editor = std::getenv("EDITOR");
+
+	string command;
+	if (editor != NULL)
+		command = ToStr(editor) + " " + mFilename;
+	else
+		command = "/usr/bin/editor " + mFilename;
+	_dbg3("Opening editor with command: " << command);
+	if ( system( command.c_str() ) == -1 )
+		_erro("Cannot execute system command: " << command);
+}
+
+const string cEnvUtils::ReadFromTmpFile() {
+	std::ifstream ifs(mFilename);
+	string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
+	return msg;
+}
+
+const string cEnvUtils::Compose() {
+	GetTmpTextFile();
+	OpenEditor();
+	string input = ReadFromTmpFile();
+	CloseFile();
+	return input;
+}
+
+#endif
+
+const string cEnvUtils::ReadFromFile(const string path) {
+	std::ifstream ifs(path);
+	string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
+	return msg;
+}
+
+void hintingToTxt(std::fstream & file, string command, vector<string> &commands) {
+	if(file.good()) {
+		file<<command<<"~"<<endl;
+		for (auto a: commands) {
+			file <<a<< " ";
+			file.flush();
+		}
+		file<<endl;
+	}
+}
+
+string stringToColor(const string &hash) {
+  // Generete vector with all possible light colors
+  vector <string> lightColors;
+  using namespace zkr;
+  lightColors.push_back(cc::fore::lightblue);
+  lightColors.push_back(cc::fore::lightred);
+  lightColors.push_back(cc::fore::lightmagenta);
+  lightColors.push_back(cc::fore::lightgreen);
+  lightColors.push_back(cc::fore::lightcyan);
+  lightColors.push_back(cc::fore::lightyellow);
+  lightColors.push_back(cc::fore::lightwhite);
+
+  int sum=0;
+
+  for (auto ch : hash) sum+=ch;
+  auto color = sum%(lightColors.size()-1);
+
+  return lightColors.at( color );
+}
+
+
+// ====================================================================
+// algorthms
+
+
+}; // namespace nUtil
+
+
+}; // namespace OT
+
+// global namespace
+
+const extern int _dbg_ignore = 0; // see description in .hpp
+
+std::string GetObjectName() {
+	//static std::string * name=nullptr;
+	//if (!name) name = new std::string("(global)");
+	return "";
+}
+
+// ====================================================================
+
+nOT::nUtils::cLogger gCurrentLogger;
+
diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp
new file mode 100644
index 000000000..6cfd11ee1
--- /dev/null
+++ b/contrib/otshell_utils/utils.hpp
@@ -0,0 +1,446 @@
+/// @file
+/// @author rfree (current maintainer in monero.cc project)
+/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
+
+/* See other files here for the LICENCE that applies here. */
+
+#include "ccolor.hpp"
+#ifndef INCLUDE_OT_NEWCLI_UTILS
+#define INCLUDE_OT_NEWCLI_UTILS
+
+#include "lib_common1.hpp"
+#ifdef __unix
+	#include <unistd.h>
+#endif
+
+#ifndef CFG_WITH_TERMCOLORS
+	#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)."
+#endif
+
+///Macros related to automatic deduction of class name etc; 
+#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; }
+#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public:
+
+namespace nOT {
+
+namespace nUtils {
+
+/// @brief general based for my runtime errors
+class myexception : public std::runtime_error {
+	public:
+		myexception(const char * what);
+		myexception(const std::string &what);
+        //virtual ~myexception();
+		virtual void Report() const;
+};
+
+/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc.
+INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1; // <=== namespaces
+
+// ======================================================================================
+/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again
+/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
+std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
+std::string & ltrim(std::string &s); ///< left trim
+std::string & rtrim(std::string &s); ///< right trim
+
+// ======================================================================================
+
+std::string get_current_time();
+
+// string conversions
+template <class T>
+std::string ToStr(const T & obj) {
+	std::ostringstream oss;
+	oss << obj;
+	return oss.str();
+}
+
+struct cNullstream : std::ostream {
+	cNullstream() : std::ios(0), std::ostream(0) {}
+};
+extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data)
+// ========== debug ==========
+// _dbg_ignore is moved to global namespace (on purpose)
+
+// TODO make _dbg_ignore thread-safe everywhere
+
+extern std::mutex gLoggerGuard;
+
+
+
+#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \
+	nOT::nUtils::gLoggerGuard.try_lock(); \
+	gCurrentLogger.write_stream(LEVEL,CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \
+	nOT::nUtils::gLoggerGuard.unlock(); \
+	} } while(0)
+
+#define _debug_level(LEVEL,VAR) _debug_level_c("",LEVEL,VAR)
+
+#define _dbg3(VAR) _debug_level( 20,VAR)
+#define _dbg2(VAR) _debug_level( 30,VAR)
+#define _dbg1(VAR) _debug_level( 40,VAR) // details
+#define _info(VAR) _debug_level( 50,VAR) // more boring info
+#define _note(VAR) _debug_level( 70,VAR) // info
+#define _fact(VAR) _debug_level( 75,VAR) // interesting event
+#define _mark(VAR) _debug_level( 80,VAR) // marked action
+#define _warn(VAR) _debug_level( 90,VAR) // some problem
+#define _erro(VAR) _debug_level(100,VAR) // error - report
+
+#define _dbg3_c(C,VAR) _debug_level_c(C, 20,VAR)
+#define _dbg2_c(C,VAR) _debug_level_c(C, 30,VAR)
+#define _dbg1_c(C,VAR) _debug_level_c(C, 40,VAR) // details
+#define _info_c(C,VAR) _debug_level_c(C, 50,VAR) // more boring info
+#define _note_c(C,VAR) _debug_level_c(C, 70,VAR) // info
+#define _fact_c(C,VAR) _debug_level_c(C, 75,VAR) // interesting event
+#define _mark_c(C,VAR) _debug_level_c(C, 80,VAR) // marked action
+#define _warn_c(C,VAR) _debug_level_c(C, 90,VAR) // some problem
+#define _erro_c(C,VAR) _debug_level_c(C,100,VAR) // error - report
+
+// lock // because od VAR
+#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \
+	std::ostringstream debug_detail_oss; \
+	nOT::nUtils::gLoggerGuard.try_lock(); \
+	debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \
+	nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \
+	if (_dbg_ignore<LEVEL) debugScopeGuard.Assign(CHANNEL,LEVEL, debug_detail_oss.str()); \
+	if (_dbg_ignore<LEVEL) _debug_level_c(CHANNEL,LEVEL,debug_detail_oss.str() + " ... begin"); \
+	nOT::nUtils::gLoggerGuard.unlock();
+#define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR)
+
+#define _scope_dbg1(VAR) _scope_debug_level( 20,VAR)
+#define _scope_dbg2(VAR) _scope_debug_level( 30,VAR)
+#define _scope_dbg3(VAR) _scope_debug_level( 40,VAR) // details
+#define _scope_info(VAR) _scope_debug_level( 50,VAR) // more boring info
+#define _scope_note(VAR) _scope_debug_level( 70,VAR) // info
+#define _scope_fact(VAR) _scope_debug_level( 75,VAR) // interesting event
+#define _scope_mark(VAR) _scope_debug_level( 80,VAR) // marked action
+#define _scope_warn(VAR) _scope_debug_level( 90,VAR) // some problem
+#define _scope_erro(VAR) _scope_debug_level( 100,VAR) // error - report
+
+/***
+@brief do not use this namespace directly, it is implementation detail.
+*/
+namespace nDetail { 
+
+/***
+@brief a Debug scope-guard, to log a debug message when current scope is left. Do NOT use this directly,
+only use it via the macros like _scope_dbg1 etc.
+*/
+class cDebugScopeGuard {
+	protected:
+		string mMsg;
+		int mLevel;
+		string mChan;
+	public:
+		cDebugScopeGuard();
+		~cDebugScopeGuard();
+		void Assign(const string &chan, const int level, const string &msg);
+};
+
+const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug
+
+}; // namespace nDetail
+
+// ========== logger ==========
+
+/*** 
+@brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly!
+@author rfree (maintainer) 
+*/
+class cLogger {
+	public:
+		cLogger();
+		~cLogger();
+		std::ostream & write_stream(int level); ///< starts a new message on given level (e.g. writes out the icon/tag) and returns stream to output to
+		std::ostream & write_stream(int level, const std::string & channel); ///< the same but with name of the debug channel
+
+		void setOutStreamFromGlobalOptions(); // set debug level, file etc - according to global Options
+		void setOutStreamFile(const std::string &fname); // switch to using this file
+		void setDebugLevel(int level); // change the debug level e.g. to mute debug from now
+
+		std::string icon(int level) const; ///< returns "icon" for given debug level. It is text, might include color controll characters
+		std::string endline() const; ///< returns string to be written at end of message
+
+	protected:
+		unique_ptr<std::ofstream> mOutfile;
+		std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream
+
+		std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class
+
+		int mLevel; ///< current debug level
+
+		std::ostream & SelectOutput(int level, const std::string & channel);
+		void OpenNewChannel(const std::string & channel);
+		std::string GetLogBaseDir() const;
+
+		std::map< std::thread::id , int > mThread2Number; // change long thread IDs into a short nice number to show
+		int mThread2Number_Biggest; // current biggest value held there (biggest key) - works as growing-only counter basically
+		int Thread2Number(const std::thread::id id); // convert the system's thread id into a nice short our id; make one if new thread
+};
+
+
+
+// ====================================================================
+// vector debug
+
+template <class T>
+std::string vectorToStr(const T & v) {
+	std::ostringstream oss;
+	for(auto rec: v) {
+		oss << rec <<",";
+		}
+	return oss.str();
+}
+
+template <class T>
+void DisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
+	std::copy( v.begin(), v.end(), std::ostream_iterator<T>(out, delim.c_str()) );
+}
+
+template <class T>
+void EndlDisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
+	out << std::endl;
+	DisplayVector(out,v,delim);
+}
+
+template <class T>
+void DisplayVectorEndl(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
+	DisplayVector(out,v,delim);
+	out << std::endl;
+}
+
+template <class T>
+void DbgDisplayVector(const std::vector<T> &v, const std::string &delim=" ") {
+	std::cerr << "[";
+	std::copy( v.begin(), v.end(), std::ostream_iterator<T>(std::cerr, delim.c_str()) );
+	std::cerr << "]";
+}
+
+string stringToColor(const string &hash);
+template <class T, class T2>
+void DisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
+	auto *no_color = zkr::cc::fore::console;
+	for(auto var : m) {
+		out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl;
+	}
+
+}
+
+template <class T, class T2>
+void EndlDisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
+	out << endl;
+	for(auto var : m) {
+		out << var.first << delim << var.second << endl;
+	}
+}
+
+template <class T, class T2>
+void DbgDisplayMap(const std::map<T, T2> &m, const std::string &delim=" ") {
+	for(auto var : m) {
+		std::cerr << var.first << delim << var.second << endl;
+	}
+}
+
+
+template <class T>
+void DbgDisplayVectorEndl(const std::vector<T> &v, const std::string &delim=" ") {
+	DbgDisplayVector(v,delim);
+	std::cerr << std::endl;
+}
+
+void DisplayStringEndl(std::ostream & out, const std::string text);
+
+bool CheckIfBegins(const std::string & beggining, const std::string & all);
+bool CheckIfEnds (std::string const & ending, std::string const & all);
+std::string SpaceFromEscape(const std::string &s);
+std::string EscapeFromSpace(const std::string &s);
+vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib);
+char GetLastChar(const std::string & str);
+std::string GetLastCharIf(const std::string & str); // TODO unicode?
+std::string EscapeString(const std::string &s);
+
+
+template <class T>
+std::string DbgVector(const std::vector<T> &v, const std::string &delim="|") {
+	std::ostringstream oss;
+	oss << "[";
+	bool first=true;
+	for(auto vElement : v) { if (!first) oss<<delim; first=false; oss <<vElement ; }
+	oss << "]";
+	//std::copy( v.begin(), v.end(), std::ostream_iterator<T>(oss, delim.c_str()) );
+	return oss.str();
+}
+
+template <class T>
+std::ostream & operator<<(std::ostream & os, const map< T, vector<T> > & obj){
+	os << "[";
+	for(auto const & elem : obj) {
+		os << " [" << elem.first << "=" << DbgVector(elem.second) << "] ";
+	}
+	os << "]";
+  return os;
+}
+
+template <class T, class T2>
+std::string DbgMap(const map<T, T2> & map) {
+	std::ostringstream oss;
+		oss << map;
+	return oss.str();
+}
+
+// ====================================================================
+// assert
+
+// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
+// Use it like this: ASRT( x>y );  with the semicolon at end, a clever trick forces this syntax :)
+#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0)
+
+void Assert(bool result, const std::string &stamp, const std::string &condition);
+
+// ====================================================================
+// advanced string
+
+const std::string GetMultiline(string endLine = "~");
+vector<string> SplitString(const string & str);
+
+bool checkPrefix(const string & str, char prefix = '^');
+
+// ====================================================================
+// nUse utils
+
+enum class eSubjectType {Account, Asset, User, Server, Unknown};
+
+string SubjectType2String(const eSubjectType & type);
+eSubjectType String2SubjectType(const string & type);
+
+// ====================================================================
+// operation on files
+
+/// @brief tools related to filesystem
+/// @author rfree (maintainer)
+class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later)
+	public:
+		static bool CreateDirTree(const std::string & dir, bool only_below=false);
+		static char GetDirSeparator(); // eg '/' or '\'
+};
+
+
+/// @brief utils to e.g. edit a file from console
+/// @author rfree (maintainer)
+class cEnvUtils {
+	int fd;
+	string mFilename;
+
+	void GetTmpTextFile();
+	void CloseFile();
+	void OpenEditor();
+	const string ReadFromTmpFile();
+public:
+	const string Compose();
+	const string ReadFromFile(const string path);
+};
+void hintingToTxt(std::fstream & file, string command, vector<string> &commands);
+void generateQuestions (std::fstream & file, string command);
+void generateAnswers (std::fstream & file, string command, vector<string> &completions);
+
+// ====================================================================
+
+namespace nOper { // nOT::nUtils::nOper
+// cool shortcut operators, like vector + vecotr operator working same as string (appending)
+// isolated to namespace because it's unorthodox ide to implement this
+
+using namespace std;
+
+// TODO use && and move?
+template <class T>
+vector<T> operator+(const vector<T> &a, const vector<T> &b) {
+	vector<T> ret = a;
+	ret.insert( ret.end() , b.begin(), b.end() );
+	return ret;
+}
+
+template <class T>
+vector<T> operator+(const T &a, const vector<T> &b) {
+	vector<T> ret(1,a);
+	ret.insert( ret.end() , b.begin(), b.end() );
+	return ret;
+}
+
+template <class T>
+vector<T> operator+(const vector<T> &a, const T &b) {
+	vector<T> b_vector(1,a);
+	return a + b_vector;
+}
+
+template <class T>
+vector<T>& operator+=(vector<T> &a, const vector<T> &b) {
+	a.insert( a.end() , b.begin(), b.end() );
+	return a;
+}
+
+// map
+template <class TK,class TV>
+map<TK,TV> operator+(const map<TK,TV> &a, const map<TK,TV> &b) {
+	map<TK,TV> ret = a;
+	for (const auto & elem : b) {
+		ret.insert(elem);
+	}
+	return ret;
+}
+
+
+} // nOT::nUtils::nOper
+
+// ====================================================================
+
+// ====================================================================
+
+// Algorithms
+
+// ====================================================================
+// ====================================================================
+
+
+/**
+@brief Special type that on creation will be initialized to have value INIT given as template argument. 
+Might be usefull e.g. to express in the declaration of class what will be the default value of member variable
+See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm
+Probably not needed when using boost in your project.
+*/
+template <class T, T INIT>
+class value_init {
+	private:
+		T data;
+	public:
+		value_init();
+
+		T& operator=(const T& v) { data=v; return *this; }
+		operator T const &() const { return data; }
+		operator T&() { return data;	}
+};
+
+template <class T, T INIT>
+value_init<T, INIT>::value_init() :	data(INIT) { }
+
+}; // namespace nUtils
+
+}; // namespace nOT
+
+
+// global namespace
+extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc
+
+std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently)
+
+const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks, 
+// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there.
+// Or to make it runtime by providing a class normal member and editing it in runtime
+
+#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]"))
+
+
+
+
+#endif
+