diff options
Diffstat (limited to 'contrib')
-rw-r--r-- | contrib/CMakeLists.txt | 3 | ||||
-rw-r--r-- | contrib/epee/include/net/abstract_tcp_server2.h | 86 | ||||
-rw-r--r-- | contrib/epee/include/net/abstract_tcp_server2.inl | 428 | ||||
-rw-r--r-- | contrib/epee/include/net/levin_protocol_handler_async.h | 32 | ||||
-rw-r--r-- | contrib/epee/include/net/net_utils_base.h | 10 | ||||
-rw-r--r-- | contrib/epee/include/storages/levin_abstract_invoke2.h | 4 | ||||
-rw-r--r-- | contrib/epee/include/syncobj.h | 12 | ||||
-rw-r--r-- | contrib/otshell_utils/CMakeLists.txt | 14 | ||||
-rw-r--r-- | contrib/otshell_utils/LICENCE.txt | 21 | ||||
-rw-r--r-- | contrib/otshell_utils/ccolor.cpp | 116 | ||||
-rw-r--r-- | contrib/otshell_utils/ccolor.hpp | 73 | ||||
-rw-r--r-- | contrib/otshell_utils/lib_common1.hpp | 52 | ||||
-rw-r--r-- | contrib/otshell_utils/runoptions.cpp | 69 | ||||
-rw-r--r-- | contrib/otshell_utils/runoptions.hpp | 58 | ||||
-rw-r--r-- | contrib/otshell_utils/utils.cpp | 804 | ||||
-rw-r--r-- | contrib/otshell_utils/utils.hpp | 532 | ||||
-rw-r--r-- | contrib/otshell_utils/windows_stream.cpp | 64 | ||||
-rw-r--r-- | contrib/otshell_utils/windows_stream.h | 20 |
18 files changed, 2265 insertions, 133 deletions
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt new file mode 100644 index 000000000..18402a61a --- /dev/null +++ b/contrib/CMakeLists.txt @@ -0,0 +1,3 @@ + +add_subdirectory(otshell_utils) + diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 6c613c5d5..3e6ea2171 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -1,3 +1,9 @@ +/** +@file +@author from CrypoNote (see copyright below; Andrey N. Sabelnikov) +@monero rfree +@brief the connection templated-class for one peer connection +*/ // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -26,7 +32,7 @@ -#ifndef _ABSTRACT_TCP_SERVER2_H_ +#ifndef _ABSTRACT_TCP_SERVER2_H_ #define _ABSTRACT_TCP_SERVER2_H_ @@ -36,6 +42,8 @@ #include <boost/noncopyable.hpp> #include <boost/shared_ptr.hpp> #include <atomic> +#include <map> +#include <memory> #include <boost/asio.hpp> #include <boost/array.hpp> @@ -46,7 +54,9 @@ #include <boost/thread/thread.hpp> #include "net_utils_base.h" #include "syncobj.h" - +#include "../../../../src/p2p/connection_basic.hpp" +#include "../../../../contrib/otshell_utils/utils.hpp" +#include "../../../../src/p2p/network_throttle-detail.hpp" #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000 @@ -61,6 +71,12 @@ namespace net_utils protected: virtual ~i_connection_filter(){} }; + + enum t_server_role { // type of the server, e.g. so that we will know how to limit it + NET = 0, // default (not used? used for misc connections maybe?) TODO + RPC = 1, // the rpc commands + P2P = 2 // to other p2p node + }; /************************************************************************/ /* */ @@ -70,13 +86,18 @@ namespace net_utils class connection : public boost::enable_shared_from_this<connection<t_protocol_handler> >, private boost::noncopyable, - public i_service_endpoint + public i_service_endpoint, + public connection_basic { public: typedef typename t_protocol_handler::connection_context t_connection_context; /// Construct a connection with the given io_service. - explicit connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter); + + explicit connection( boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, + std::atomic<long> &ref_sock_count, // the ++/-- counter + std::atomic<long> &sock_number, // the only increasing ++ number generator + i_connection_filter * &pfilter); virtual ~connection(); /// Get the socket associated with the connection. @@ -90,7 +111,8 @@ namespace net_utils void call_back_starter(); private: //----------------- i_service_endpoint --------------------- - virtual bool do_send(const void* ptr, size_t cb); + virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) + virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data virtual bool close(); virtual bool call_run_once_service_io(); virtual bool request_callback(); @@ -107,29 +129,31 @@ namespace net_utils /// Handle completion of a write operation. void handle_write(const boost::system::error_code& e, size_t cb); - /// Strand to ensure the connection's handlers are not called concurrently. - boost::asio::io_service::strand strand_; - - /// Socket for the connection. - boost::asio::ip::tcp::socket socket_; - /// Buffer for incoming data. boost::array<char, 8192> buffer_; + //boost::array<char, 1024> buffer_; t_connection_context context; - volatile uint32_t m_want_close_connection; - std::atomic<bool> m_was_shutdown; - critical_section m_send_que_lock; - std::list<std::string> m_send_que; - volatile uint32_t& m_ref_sockets_count; i_connection_filter* &m_pfilter; - volatile bool m_is_multithreaded; + // TODO what do they mean about wait on destructor?? --rfree : //this should be the last one, because it could be wait on destructor, while other activities possible on other threads t_protocol_handler m_protocol_handler; //typename t_protocol_handler::config_type m_dummy_config; std::list<boost::shared_ptr<connection<t_protocol_handler> > > m_self_refs; // add_ref/release support critical_section m_self_refs_lock; + critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk() + + t_server_role m_connection_type; + + // for calculate speed (last 60 sec) + network_throttle m_throttle_speed_in; + network_throttle m_throttle_speed_out; + std::mutex m_throttle_speed_in_mutex; + std::mutex m_throttle_speed_out_mutex; + + public: + void setRPcStation(); }; @@ -146,9 +170,11 @@ namespace net_utils /// Construct the server to listen on the specified TCP address and port, and /// serve up files from the given directory. boosted_tcp_server(); - explicit boosted_tcp_server(boost::asio::io_service& external_io_service); + explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_server_role s_type); ~boosted_tcp_server(); - + + std::map<std::string, t_server_role> server_type_map; + void create_server_type_map(); bool init_server(uint32_t port, const std::string address = "0.0.0.0"); bool init_server(const std::string port, const std::string& address = "0.0.0.0"); @@ -254,22 +280,26 @@ namespace net_utils /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; - /// The next connection to be accepted. - connection_ptr new_connection_; std::atomic<bool> m_stop_signal_sent; uint32_t m_port; - volatile uint32_t m_sockets_count; + std::atomic<long> m_sock_count; + std::atomic<long> m_sock_number; std::string m_address; - std::string m_thread_name_prefix; + std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; i_connection_filter* m_pfilter; std::vector<boost::shared_ptr<boost::thread> > m_threads; boost::thread::id m_main_thread_id; critical_section m_threads_lock; - volatile uint32_t m_thread_index; - }; -} -} + volatile uint32_t m_thread_index; // TODO change to std::atomic + t_server_role type; + void detach_threads(); + + /// The next connection to be accepted + connection_ptr new_connection_; + }; // class <>boosted_tcp_server +} // namespace +} // namespace #include "abstract_tcp_server2.inl" diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index db3f9e322..31836fe9e 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -1,3 +1,9 @@ +/** +@file +@author from CrypoNote (see copyright below; Andrey N. Sabelnikov) +@monero rfree +@brief the connection templated-class for one peer connection +*/ // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -26,7 +32,7 @@ -#include "net_utils_base.h" +//#include "net_utils_base.h" #include <boost/lambda/bind.hpp> #include <boost/foreach.hpp> #include <boost/lambda/lambda.hpp> @@ -34,9 +40,21 @@ #include <boost/chrono.hpp> #include <boost/utility/value_init.hpp> #include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> // TODO +#include <boost/thread/thread.hpp> // TODO #include "misc_language.h" #include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + +#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +#include "../../../../contrib/otshell_utils/utils.hpp" +#include "../../../../src/p2p/data_logger.hpp" +using namespace nOT::nUtils; // TODO + PRAGMA_WARNING_PUSH namespace epee { @@ -48,17 +66,21 @@ namespace net_utils PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> - connection<t_protocol_handler>::connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter) - : strand_(io_service), - socket_(io_service), - m_want_close_connection(0), - m_was_shutdown(0), - m_ref_sockets_count(sock_count), - m_pfilter(pfilter), - m_protocol_handler(this, config, context) + connection<t_protocol_handler>::connection( boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, + std::atomic<long> &ref_sock_count, // the ++/-- counter + std::atomic<long> &sock_number, // the only increasing ++ number generator + i_connection_filter* &pfilter + ) + : + connection_basic(io_service, ref_sock_count, sock_number), + m_protocol_handler(this, config, context), + m_pfilter( pfilter ), + m_connection_type(NET), + m_throttle_speed_in("speed_in", "throttle_speed_in"), + m_throttle_speed_out("speed_out", "throttle_speed_out") { - boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count); + _info_c("net/sleepRPC", "connection constructor set m_connection_type="<<m_connection_type); } PRAGMA_WARNING_DISABLE_VS(4355) //--------------------------------------------------------------------------------- @@ -67,12 +89,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) { if(!m_was_shutdown) { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); + _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); shutdown(); } - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed"); - boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count); + _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed"); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -118,13 +139,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income); - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << + _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << " to " << local_ep.address().to_string() << ':' << local_ep.port() << - ", total sockets objects " << m_ref_sockets_count); + ", total sockets objects " << m_ref_sock_count); if(m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip)) { - LOG_PRINT_L2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); + _dbg2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); close(); return false; } @@ -136,7 +157,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::bind(&connection<t_protocol_handler>::handle_read, self, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - + + //set ToS flag + int tos = get_tos_flag(); + boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS > + optionTos( tos ); + socket_.set_option( optionTos ); + //_dbg1("Set ToS flag to " << tos); + + boost::asio::ip::tcp::no_delay noDelayOption(false); + socket_.set_option(noDelayOption); + return true; CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false); @@ -146,7 +177,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection<t_protocol_handler>::request_callback() { TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback"); + _dbg2("[" << print_connection_context_short(context) << "] request_callback"); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted auto self = safe_shared_from_this(); if(!self) @@ -167,8 +198,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection<t_protocol_handler>::add_ref() { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref"); + //_dbg3("[sock " << socket_.native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); CRITICAL_REGION_LOCAL(m_self_refs_lock); + //_dbg3("[sock " << socket_.native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted auto self = safe_shared_from_this(); @@ -201,7 +233,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) void connection<t_protocol_handler>::call_back_starter() { TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback"); + _dbg2("[" << print_connection_context_short(context) << "] fired_callback"); m_protocol_handler.handle_qued_callback(); CATCH_ENTRY_L0("connection<t_protocol_handler>::call_back_starter()", void()); } @@ -211,17 +243,44 @@ PRAGMA_WARNING_DISABLE_VS(4355) std::size_t bytes_transferred) { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async read calledback."); + //_info("[sock " << socket_.native_handle() << "] Async read calledback."); if (!e) { - LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4); + { + CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex); + m_throttle_speed_in.handle_trafic_exact(bytes_transferred); + context.m_current_speed_down = m_throttle_speed_in.get_current_speed(); + } + + { + CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); + epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred * 1024); + } + double delay=0; // will be calculated + do + { + { //_scope_dbg1("CRITICAL_REGION_LOCAL"); + CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); + delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); // decission from global + } + + delay *= 0.5; + if (delay > 0) { + long int ms = (long int)(delay * 100); + epee::net_utils::data_logger::get_instance().add_data("sleep_down", ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } while(delay > 0); + + //_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred); + logger_handle_net_read(bytes_transferred); context.m_last_recv = time(NULL); context.m_recv_cnt += bytes_transferred; bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); if(!recv_res) { - LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] protocol_want_close"); //some error in protocol, protocol handler ask to close connection boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); @@ -239,14 +298,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::bind(&connection<t_protocol_handler>::handle_read, connection<t_protocol_handler>::shared_from_this(), boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Async read requested."); + //_info("[sock " << socket_.native_handle() << "]Async read requested."); } }else { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); if(e.value() != 2) { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); shutdown(); } } @@ -283,8 +342,91 @@ PRAGMA_WARNING_DISABLE_VS(4355) CATCH_ENTRY_L0("connection<t_protocol_handler>::call_run_once_service_io", false); } //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) { + TRY_ENTRY(); + + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if (!self) return false; + if (m_was_shutdown) return false; + // TODO avoid copy + + const double factor = 32; // TODO config + typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic + const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) ); + const t_safe chunksize_max = chunksize_good * 2 ; + const bool allow_split = (m_connection_type == RPC) ? false : true; // TODO config + + ASRT(! (chunksize_max<0) ); // make sure it is unsigned before removin sign with cast: + long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ; + + if (allow_split && (cb > chunksize_max_unsigned)) { + { // LOCK: chunking + epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** + + _dbg3_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr); + t_safe all = cb; // all bytes to send + t_safe pos = 0; // current sending position + // 01234567890 + // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 + // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 + // ^^^ (pos=8, len=4) ; + + // const size_t bufsize = chunksize_good; // TODO safecast + // char* buf = new char[ bufsize ]; + + bool all_ok = true; + while (pos < all) { + t_safe lenall = all-pos; // length from here to end + t_safe len = std::min( chunksize_good , lenall); // take a smaller part + ASRT(len<=chunksize_good); + // pos=8; len=4; all=10; len=3; + + ASRT(! (len<0) ); // check before we cast away sign: + unsigned long long int len_unsigned = static_cast<long long int>( len ); + ASRT(len>0); // (redundand) + ASRT(len_unsigned < std::numeric_limits<size_t>::max()); // yeap we want strong < then max size, to be sure + + void *chunk_start = ((char*)ptr) + pos; + _fact_c("net/out/size","chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos); + ASRT(chunk_start >= ptr); // not wrapped around address? + //std::memcpy( (void*)buf, chunk_start, len); + + _dbg3_c("net/out/size", "part of " << lenall << ": pos="<<pos << " len="<<len); + + bool ok = do_send_chunk(chunk_start, len); // <====== *** + + all_ok = all_ok && ok; + if (!all_ok) { + _dbg1_c("net/out/size", "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr); + _dbg1("do_send() SEND was aborted in middle of big package - this is mostly harmless " + << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb); + return false; // partial failure in sending + } + pos = pos+len; ASRT(pos >0); + + // (in catch block, or uniq pointer) delete buf; + } // each chunk + + _dbg3_c("net/out/size", "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr); + _dbg3 ( "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr); + + _info_c("net/sleepRPC", "do_send() m_connection_type = " << m_connection_type); + + return all_ok; // done - e.g. queued - all the chunks of current do_send call + } // LOCK: chunking + } // a big block (to be chunked) - all chunks + else { // small block + return do_send_chunk(ptr,cb); // just send as 1 big chunk + } + + CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); + } // do_send() + + //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) + bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -293,50 +435,86 @@ PRAGMA_WARNING_DISABLE_VS(4355) return false; if(m_was_shutdown) return false; + { + CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); + m_throttle_speed_out.handle_trafic_exact(cb); + context.m_current_speed_up = m_throttle_speed_out.get_current_speed(); + } - LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); context.m_send_cnt += cb; //some data should be wrote to stream //request complete - epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); - if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) + sleep_before_packet(cb, 1, 1); + epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); // *** critical *** + long int retry=0; + const long int retry_limit = 5*4; + while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) { - send_guard.unlock(); - LOG_PRINT_L2("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); - close(); - return false; + retry++; + + /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop + _fact("ABORT queue wait due to stopping"); + return false; // aborted + }*/ + + long int ms = 250 + (rand()%50); + _info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); + _dbg1("sleep for queue: " << ms); + + if (retry > retry_limit) { + send_guard.unlock(); + _erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + // _dbg1_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + close(); + return false; + } } m_send_que.resize(m_send_que.size()+1); m_send_que.back().assign((const char*)ptr, cb); if(m_send_que.size() > 1) - { - //active operation should be in progress, nothing to do, just wait last operation callback - }else - { - //no active operation - if(m_send_que.size()!=1) - { - LOG_ERROR("Looks like no active operations, but send que size != 1!!"); - return false; - } - - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) - //) - ); + { // active operation should be in progress, nothing to do, just wait last operation callback + auto size_now = cb; + _info_c("net/out/size", "do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size()); + //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); } + else + { // no active operation + + if(m_send_que.size()!=1) + { + _erro("Looks like no active operations, but send que size != 1!!"); + return false; + } + + auto size_now = m_send_que.front().size(); + _dbg1_c("net/out/size", "do_send() NOW SENSD: packet="<<size_now<<" B"); + do_send_handler_write( ptr , size_now ); // (((H))) + + ASRT( size_now == m_send_que.front().size() ); + boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) , + //strand_.wrap( + boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) + //) + ); + //_dbg3("(chunk): " << size_now); + //logger_handle_net_write(size_now); + //_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); + } + + //do_send_handler_stop( ptr , cb ); // empty function return true; CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); - } + } // do_send_chunk //--------------------------------------------------------------------------------- template<class t_protocol_handler> bool connection<t_protocol_handler>::shutdown() @@ -353,7 +531,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection<t_protocol_handler>::close() { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called."); + //_info("[sock " << socket_.native_handle() << "] Que Shutdown called."); size_t send_que_size = 0; CRITICAL_REGION_BEGIN(m_send_que_lock); send_que_size = m_send_que.size(); @@ -376,16 +554,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (e) { - LOG_PRINT_L1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); + _dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); shutdown(); return; } - + logger_handle_net_write(cb); + sleep_before_packet(cb, 1, 1); bool do_shutdown = false; CRITICAL_REGION_BEGIN(m_send_que_lock); if(m_send_que.empty()) { - LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); + _erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); return; } @@ -399,10 +578,16 @@ PRAGMA_WARNING_DISABLE_VS(4355) }else { //have more data to send - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)); - //); + auto size_now = m_send_que.front().size(); + _dbg1_c("net/out/size", "handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from queue size="<<m_send_que.size()); + do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H))) + ASRT( size_now == m_send_que.front().size() ); + boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now) , + // strand_.wrap( + boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2) + // ) + ); + //_dbg3("(normal)" << size_now); } CRITICAL_REGION_END(); @@ -412,6 +597,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) } CATCH_ENTRY_L0("connection<t_protocol_handler>::handle_write", void()); } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void connection<t_protocol_handler>::setRPcStation() + { + m_connection_type = RPC; + _fact_c("net/sleepRPC", "set m_connection_type = RPC "); + } /************************************************************************/ /* */ /************************************************************************/ @@ -420,19 +612,27 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_io_service_local_instance(new boost::asio::io_service()), io_service_(*m_io_service_local_instance.get()), acceptor_(io_service_), - new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) + m_stop_signal_sent(false), m_port(0), + m_sock_count(0), m_sock_number(0), m_threads_count(0), + m_pfilter(NULL), m_thread_index(0), + new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter)) { + create_server_type_map(); m_thread_name_prefix = "NET"; + type = NET; } template<class t_protocol_handler> - boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service): + boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_server_role s_type): io_service_(extarnal_io_service), acceptor_(io_service_), - new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) + m_stop_signal_sent(false), m_port(0), + m_sock_count(0), m_sock_number(0), m_threads_count(0), + m_pfilter(NULL), m_thread_index(0), + type(NET), + new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter)) { + create_server_type_map(); m_thread_name_prefix = "NET"; } //--------------------------------------------------------------------------------- @@ -444,6 +644,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::create_server_type_map() + { + server_type_map["NET"] = t_server_role::NET; + server_type_map["RPC"] = t_server_role::RPC; + server_type_map["P2P"] = t_server_role::P2P; + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address) { TRY_ENTRY(); @@ -491,6 +699,7 @@ POP_WARNINGS std::string thread_name = std::string("[") + m_thread_name_prefix; thread_name += boost::to_string(local_thr_index) + "]"; log_space::log_singletone::set_thread_log_prefix(thread_name); + // _fact("Thread name: " << m_thread_name_prefix); while(!m_stop_signal_sent) { try @@ -499,14 +708,14 @@ POP_WARNINGS } catch(const std::exception& ex) { - LOG_ERROR("Exception at server worker thread, what=" << ex.what()); + _erro("Exception at server worker thread, what=" << ex.what()); } catch(...) { - LOG_ERROR("Exception at server worker thread, unknown execption"); + _erro("Exception at server worker thread, unknown execption"); } } - LOG_PRINT_L4("Worker thread finished"); + //_info("Worker thread finished"); return true; CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::worker_thread", false); } @@ -515,6 +724,9 @@ POP_WARNINGS void boosted_tcp_server<t_protocol_handler>::set_threads_prefix(const std::string& prefix_name) { m_thread_name_prefix = prefix_name; + type = server_type_map[m_thread_name_prefix]; + _note("Set server type to: " << type); + _note("Set server type to: " << m_thread_name_prefix); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -539,32 +751,38 @@ POP_WARNINGS { boost::shared_ptr<boost::thread> thread(new boost::thread( attrs, boost::bind(&boosted_tcp_server<t_protocol_handler>::worker_thread, this))); + _note("Run server thread name: " << m_thread_name_prefix); m_threads.push_back(thread); } CRITICAL_REGION_END(); // Wait for all threads in the pool to exit. - if(wait) + if (wait) // && ! ::cryptonote::core::get_is_stopping()) // TODO fast_exit { - for (std::size_t i = 0; i < m_threads.size(); ++i) - m_threads[i]->join(); + _fact("JOINING all threads"); + for (std::size_t i = 0; i < m_threads.size(); ++i) { + m_threads[i]->join(); + } + _fact("JOINING all threads - almost"); m_threads.clear(); + _fact("JOINING all threads - DONE"); - }else - { + } + else { + _dbg1("Reiniting OK."); return true; } if(wait && !m_stop_signal_sent) { //some problems with the listening socket ?.. - LOG_PRINT_L0("Net service stopped without stop request, restarting..."); + _dbg1("Net service stopped without stop request, restarting..."); if(!this->init_server(m_port, m_address)) { - LOG_PRINT_L0("Reiniting service failed, exit."); + _dbg1("Reiniting service failed, exit."); return false; }else { - LOG_PRINT_L0("Reiniting OK."); + _dbg1("Reiniting OK."); } } } @@ -597,7 +815,7 @@ POP_WARNINGS { if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms)) { - LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle()); + _dbg1("Interrupting thread " << m_threads[i]->native_handle()); m_threads[i]->interrupt(); } } @@ -608,6 +826,10 @@ POP_WARNINGS template<class t_protocol_handler> void boosted_tcp_server<t_protocol_handler>::send_stop_signal() { + if (::cryptonote::core::get_fast_exit() == true) + { + detach_threads(); + } m_stop_signal_sent = true; TRY_ENTRY(); io_service_.stop(); @@ -626,19 +848,22 @@ POP_WARNINGS TRY_ENTRY(); if (!e) { - connection_ptr conn(std::move(new_connection_)); - - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)); + if (type == RPC) { + new_connection_->setRPcStation(); + _note("New server for RPC connections"); + } + connection_ptr conn(std::move(new_connection_)); + new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter)); acceptor_.async_accept(new_connection_->socket(), boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, boost::asio::placeholders::error)); bool r = conn->start(true, 1 < m_threads_count); if (!r) - LOG_ERROR("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count); + _erro("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); }else { - LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count); + _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count); } CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::handle_accept", void()); } @@ -648,7 +873,7 @@ POP_WARNINGS { TRY_ENTRY(); - connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) ); + connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) ); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); ////////////////////////////////////////////////////////////////////////// @@ -658,7 +883,7 @@ POP_WARNINGS boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - LOG_ERROR("Failed to resolve " << adr); + _erro("Failed to resolve " << adr); return false; } ////////////////////////////////////////////////////////////////////////// @@ -704,7 +929,7 @@ POP_WARNINGS { //timeout sock_.close(); - LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); + _dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); return false; } } @@ -712,21 +937,21 @@ POP_WARNINGS if (ec || !sock_.is_open()) { - LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3); + _dbg3("Some problems at connect, message: " << ec.message()); return false; } - LOG_PRINT_L3("Connected success to " << adr << ':' << port); + _dbg3("Connected success to " << adr << ':' << port); bool r = new_connection_l->start(false, 1 < m_threads_count); if (r) { new_connection_l->get_context(conn_context); - //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)); + //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_pfilter)); } else { - LOG_ERROR("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count); + _erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); } return r; @@ -738,7 +963,7 @@ POP_WARNINGS bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip) { TRY_ENTRY(); - connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) ); + connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter) ); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); ////////////////////////////////////////////////////////////////////////// @@ -748,7 +973,7 @@ POP_WARNINGS boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - LOG_ERROR("Failed to resolve " << adr); + _erro("Failed to resolve " << adr); return false; } ////////////////////////////////////////////////////////////////////////// @@ -768,7 +993,7 @@ POP_WARNINGS { if(error != boost::asio::error::operation_aborted) { - LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); + _dbg3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); new_connection_l->socket().close(); } }); @@ -785,7 +1010,7 @@ POP_WARNINGS cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation }else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port()); bool r = new_connection_l->start(false, 1 < m_threads_count); if (r) @@ -795,13 +1020,13 @@ POP_WARNINGS } else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port); + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port); cb(conn_context, boost::asio::error::fault); } } }else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value()); cb(conn_context, ec_); } @@ -809,6 +1034,15 @@ POP_WARNINGS return true; CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::connect_async", false); } -} -} + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::detach_threads() + { + for (auto thread : m_threads) + thread->detach(); + } + + +} // namespace +} // namespace PRAGMA_WARNING_POP diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index e79768b04..a7fbffb4b 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -34,6 +34,9 @@ #include "levin_base.h" #include "misc_language.h" +#include <random> +#include <chrono> + namespace epee { @@ -81,6 +84,7 @@ public: async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE) {} + void del_out_connections(size_t count); }; @@ -669,6 +673,34 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p } //------------------------------------------------------------------------------------------ template<class t_connection_context> +void async_protocol_handler_config<t_connection_context>::del_out_connections(size_t count) +{ + std::vector <boost::uuids::uuid> out_connections; + CRITICAL_REGION_BEGIN(m_connects_lock); + for (auto& c: m_connects) + { + if (!c.second->m_connection_context.m_is_income) + out_connections.push_back(c.first); + } + + if (out_connections.size() == 0) + return; + + // close random out connections + // TODO or better just keep removing random elements (performance) + unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); + shuffle(out_connections.begin(), out_connections.end(), std::default_random_engine(seed)); + while (count > 0 && out_connections.size() > 0) + { + close(*out_connections.begin()); + del_connection(m_connects.at(*out_connections.begin())); + --count; + } + + CRITICAL_REGION_END(); +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::add_connection(async_protocol_handler<t_connection_context>* pconn) { CRITICAL_REGION_BEGIN(m_connects_lock); diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 90e352787..f963e7746 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -55,6 +55,8 @@ namespace net_utils time_t m_last_send; uint64_t m_recv_cnt; uint64_t m_send_cnt; + double m_current_speed_down; + double m_current_speed_up; connection_context_base(boost::uuids::uuid connection_id, long remote_ip, int remote_port, bool is_income, @@ -68,7 +70,9 @@ namespace net_utils m_last_recv(last_recv), m_last_send(last_send), m_recv_cnt(recv_cnt), - m_send_cnt(send_cnt) + m_send_cnt(send_cnt), + m_current_speed_down(0), + m_current_speed_up(0) {} connection_context_base(): m_connection_id(), @@ -79,7 +83,9 @@ namespace net_utils m_last_recv(0), m_last_send(0), m_recv_cnt(0), - m_send_cnt(0) + m_send_cnt(0), + m_current_speed_down(0), + m_current_speed_up(0) {} connection_context_base& operator=(const connection_context_base& a) diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 1b32c51d1..73ede1b12 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -185,7 +185,7 @@ namespace epee } return res; - }; + } template<class t_owner, class t_in_type, class t_context, class callback_t> int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context) @@ -199,7 +199,7 @@ namespace epee boost::value_initialized<t_in_type> in_struct; static_cast<t_in_type&>(in_struct).load(strg); return cb(command, in_struct, context); - }; + } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \ diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index b7273da8e..b81eb43a9 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -35,10 +35,14 @@ #include <boost/thread/locks.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> +#include <thread> +#include <chrono> namespace epee { + extern unsigned int g_test_dbg_lock_sleep; + struct simple_event { simple_event() : m_rised(false) @@ -215,10 +219,10 @@ namespace epee #define SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard critical_region_var(x) -#define CRITICAL_REGION_LOCAL(x) epee::critical_region_t<decltype(x)> critical_region_var(x) -#define CRITICAL_REGION_BEGIN(x) { epee::critical_region_t<decltype(x)> critical_region_var(x) -#define CRITICAL_REGION_LOCAL1(x) epee::critical_region_t<decltype(x)> critical_region_var1(x) -#define CRITICAL_REGION_BEGIN1(x) { epee::critical_region_t<decltype(x)> critical_region_var1(x) +#define CRITICAL_REGION_LOCAL(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_BEGIN(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_LOCAL1(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var1(x) +#define CRITICAL_REGION_BEGIN1(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var1(x) #define CRITICAL_REGION_END() } diff --git a/contrib/otshell_utils/CMakeLists.txt b/contrib/otshell_utils/CMakeLists.txt new file mode 100644 index 000000000..7413e0dc5 --- /dev/null +++ b/contrib/otshell_utils/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required (VERSION 2.6) +project (otshell CXX) + +# Add executable + +file(GLOB otshell_utils_sources # All files in directory: + "*.h" + "*.hpp" + "*.cpp" +) + +add_library (otshell_utils STATIC ${otshell_utils_sources}) +set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils") +#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs diff --git a/contrib/otshell_utils/LICENCE.txt b/contrib/otshell_utils/LICENCE.txt new file mode 100644 index 000000000..f351acf10 --- /dev/null +++ b/contrib/otshell_utils/LICENCE.txt @@ -0,0 +1,21 @@ + +This are some files also from OpenTransactions / otshell project, +developed thanks to the awesome OpenTransaction project, organization and developers :) + +Parts of code here was also developed thanks to the excellent Monero project, +thanks to Monero project, organization and developers :) + +[Some] files/code here (in external/otshell_utils) are under licence defined in +src/doc/LICENCE-otshell.txt ; +Others are from monero, with licence in src/doc/LICENCE-monero.txt ; + +For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code, +possibly not all) I do not worry who uses it how; I'am not a lawyer. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project +Monero and Open Transactions, to share and develop both faster. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + diff --git a/contrib/otshell_utils/ccolor.cpp b/contrib/otshell_utils/ccolor.cpp new file mode 100644 index 000000000..cd93e0de7 --- /dev/null +++ b/contrib/otshell_utils/ccolor.cpp @@ -0,0 +1,116 @@ +#include "ccolor.hpp" +#include <cstdarg> + +// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal +// from http://wiznet.gr/src/ccolor.zip +// edited by rfree - as part of https://github.com/rfree/Open-Transactions/ + +using namespace std; + +#ifdef _MSC_VER + +#define snprintf c99_snprintf + +inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) { + int count = -1; + if (size != 0) + count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); + if (count == -1) + count = _vscprintf(format, ap); + return count; +} + +inline int c99_snprintf(char* str, size_t size, const char* format, ...) { + int count; + va_list ap; + va_start(ap, format); + count = c99_vsnprintf(str, size, format, ap); + va_end(ap); + return count; +} +#endif // _MSC_VER + +#define CC_CONSOLE_COLOR_DEFAULT "\033[0m" +#define CC_FORECOLOR(C) "\033[" #C "m" +#define CC_BACKCOLOR(C) "\033[" #C "m" +#define CC_ATTR(A) "\033[" #A "m" + +namespace zkr +{ + enum Color + { + Black, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, + White, + Default = 9 + }; + + enum Attributes + { + Reset, + Bright, + Dim, + Underline, + Blink, + Reverse, + Hidden + }; + + char * cc::color(int attr, int fg, int bg) + { + static const int size = 20; + static char command[size]; + + /* Command is the control command to the terminal */ + snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40); + return command; + } + + + const char *cc::console = CC_CONSOLE_COLOR_DEFAULT; + const char *cc::underline = CC_ATTR(4); + const char *cc::bold = CC_ATTR(1); + + const char *cc::fore::black = CC_FORECOLOR(30); + const char *cc::fore::blue = CC_FORECOLOR(34); + const char *cc::fore::red = CC_FORECOLOR(31); + const char *cc::fore::magenta = CC_FORECOLOR(35); + const char *cc::fore::green = CC_FORECOLOR(92); + const char *cc::fore::cyan = CC_FORECOLOR(36); + const char *cc::fore::yellow = CC_FORECOLOR(33); + const char *cc::fore::white = CC_FORECOLOR(37); + const char *cc::fore::console = CC_FORECOLOR(39); + + const char *cc::fore::lightblack = CC_FORECOLOR(90); + const char *cc::fore::lightblue = CC_FORECOLOR(94); + const char *cc::fore::lightred = CC_FORECOLOR(91); + const char *cc::fore::lightmagenta = CC_FORECOLOR(95); + const char *cc::fore::lightgreen = CC_FORECOLOR(92); + const char *cc::fore::lightcyan = CC_FORECOLOR(96); + const char *cc::fore::lightyellow = CC_FORECOLOR(93); + const char *cc::fore::lightwhite = CC_FORECOLOR(97); + + const char *cc::back::black = CC_BACKCOLOR(40); + const char *cc::back::blue = CC_BACKCOLOR(44); + const char *cc::back::red = CC_BACKCOLOR(41); + const char *cc::back::magenta = CC_BACKCOLOR(45); + const char *cc::back::green = CC_BACKCOLOR(42); + const char *cc::back::cyan = CC_BACKCOLOR(46); + const char *cc::back::yellow = CC_BACKCOLOR(43); + const char *cc::back::white = CC_BACKCOLOR(47); + const char *cc::back::console = CC_BACKCOLOR(49); + + const char *cc::back::lightblack = CC_BACKCOLOR(100); + const char *cc::back::lightblue = CC_BACKCOLOR(104); + const char *cc::back::lightred = CC_BACKCOLOR(101); + const char *cc::back::lightmagenta = CC_BACKCOLOR(105); + const char *cc::back::lightgreen = CC_BACKCOLOR(102); + const char *cc::back::lightcyan = CC_BACKCOLOR(106); + const char *cc::back::lightyellow = CC_BACKCOLOR(103); + const char *cc::back::lightwhite = CC_BACKCOLOR(107); +} diff --git a/contrib/otshell_utils/ccolor.hpp b/contrib/otshell_utils/ccolor.hpp new file mode 100644 index 000000000..bf5a601a2 --- /dev/null +++ b/contrib/otshell_utils/ccolor.hpp @@ -0,0 +1,73 @@ +// ccolor.hpp + +// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal +// from http://wiznet.gr/src/ccolor.zip +// edited by rfree - as part of https://github.com/rfree/Open-Transactions/ + +#ifndef INCLUDE_OT_ccolor +#define INCLUDE_OT_ccolor + +#include <iostream> +#include <stdio.h> + +namespace zkr +{ + class cc + { + public: + + class fore + { + public: + static const char *black; + static const char *blue; + static const char *red; + static const char *magenta; + static const char *green; + static const char *cyan; + static const char *yellow; + static const char *white; + static const char *console; + + static const char *lightblack; + static const char *lightblue; + static const char *lightred; + static const char *lightmagenta; + static const char *lightgreen; + static const char *lightcyan; + static const char *lightyellow; + static const char *lightwhite; + }; + + class back + { + public: + static const char *black; + static const char *blue; + static const char *red; + static const char *magenta; + static const char *green; + static const char *cyan; + static const char *yellow; + static const char *white; + static const char *console; + + static const char *lightblack; + static const char *lightblue; + static const char *lightred; + static const char *lightmagenta; + static const char *lightgreen; + static const char *lightcyan; + static const char *lightyellow; + static const char *lightwhite; + }; + + static char *color(int attr, int fg, int bg); + static const char *console; + static const char *underline; + static const char *bold; + }; +} + +#endif + diff --git a/contrib/otshell_utils/lib_common1.hpp b/contrib/otshell_utils/lib_common1.hpp new file mode 100644 index 000000000..456e63fbb --- /dev/null +++ b/contrib/otshell_utils/lib_common1.hpp @@ -0,0 +1,52 @@ +/* See other files here for the LICENCE that applies here. */ + + +#ifndef INCLUDE_OT_NEWCLI_COMMON1 +#define INCLUDE_OT_NEWCLI_COMMON1 + +#include <string> +#include <cstring> +#include <vector> +#include <map> +#include <list> +#include <algorithm> +#include <iostream> +#include <fstream> +#include <sstream> +#include <set> +#include <iterator> +#include <stdexcept> + +#include <functional> +#include <memory> +#include <thread> +#include <atomic> +#include <mutex> + + +// list of thigs from libraries that we pull into namespace nOT::nNewcli +// we might still need to copy/paste it in few places to make IDEs pick it up correctly +#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \ + using std::string; \ + using std::vector; \ + using std::vector; \ + using std::list; \ + using std::set; \ + using std::map; \ + using std::ostream; \ + using std::istream; \ + using std::cin; \ + using std::cerr; \ + using std::cout; \ + using std::cerr; \ + using std::endl; \ + using std::function; \ + using std::unique_ptr; \ + using std::shared_ptr; \ + using std::weak_ptr; \ + using std::enable_shared_from_this; \ + using std::mutex; \ + using std::lock_guard; \ + +#endif + diff --git a/contrib/otshell_utils/runoptions.cpp b/contrib/otshell_utils/runoptions.cpp new file mode 100644 index 000000000..ffd37eae4 --- /dev/null +++ b/contrib/otshell_utils/runoptions.cpp @@ -0,0 +1,69 @@ +/* See other files here for the LICENCE that applies here. */ +/* See header file .hpp for info */ + +#include "runoptions.hpp" + +#include "lib_common1.hpp" + +namespace nOT { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +// (no debug - this is the default) +// +nodebug (no debug) +// +debug ...... --asdf +// +debug +debugcerr .... --asfs +// +debug +debugfile .... --asfs + +cRunOptions::cRunOptions() + : mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false) + ,mDoRunDebugshow(false) +{ } + +vector<string> cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector<string> & args) { + vector<string> arg_clear; // will store only the arguments that are not removed + + for (auto arg : args) { + bool thisIsRunoption=false; + + if (arg.size()>0) { + if (arg.at(0) == '+') thisIsRunoption=true; + } + + if (thisIsRunoption) Exec(arg); // *** + if (! thisIsRunoption) arg_clear.push_back(arg); + } + + Normalize(); + + return arg_clear; +} + +void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug"); + if (runoption == "+nodebug") { mDebug=false; } + else if (runoption == "+debug") { mDebug=true; } + else if (runoption == "+debugcerr") { mDebug=true; mDebugSendToCerr=true; } + else if (runoption == "+debugfile") { mDebug=true; mDebugSendToFile=true; } + else if (runoption == "+demo") { mRunMode=eRunModeDemo; } + else if (runoption == "+normal") { mRunMode=eRunModeNormal; } + else if (runoption == "+current") { mRunMode=eRunModeCurrent; } + else if (runoption == "+debugshow") { mDebug=true; mDebugSendToCerr=true; mDoRunDebugshow=true; } + else { + cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl; + throw std::runtime_error("Unknown runoption"); + } + // cerr<<"debug="<<mDebug<<endl; +} + +void cRunOptions::Normalize() { + if (mDebug) { + if (!( mDebugSendToFile || mDebugSendToCerr )) mDebugSendToCerr=true; // if debug is on then send to something, e.g. to cerr + } +} + + +cRunOptions gRunOptions; // (extern) + +} // namespace OT + + diff --git a/contrib/otshell_utils/runoptions.hpp b/contrib/otshell_utils/runoptions.hpp new file mode 100644 index 000000000..219d3b509 --- /dev/null +++ b/contrib/otshell_utils/runoptions.hpp @@ -0,0 +1,58 @@ +/* See other files here for the LICENCE that applies here. */ +/* +Template for new files, replace word "template" and later delete this line here. +*/ + +#ifndef INCLUDE_OT_NEWCLI_runoptions_hpp +#define INCLUDE_OT_NEWCLI_runoptions_hpp + +#include "lib_common1.hpp" + +namespace nOT { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +/** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug. +This is NOT for all the other options that are parsed and executed by program. */ +class cRunOptions { + public: + enum tRunMode { ///< Type of run mode - is this normal, or demonstration etc. + eRunModeCurrent=1, ///< currently developed version + eRunModeDemo, ///< best currently available Demo of something nice + eRunModeNormal, ///< do the normal things that the program should do + }; + + private: + tRunMode mRunMode; ///< selected run mode + + bool mDebug; // turn debug on, Eg: +debug without it probably nothing will be written to debug (maybe just error etc) + bool mDebugSendToFile; // send to file, Eg: for +debugfile ; also turns on debug + bool mDebugSendToCerr; // send to cerr, Eg: for +debugcerr ; also turns on debug + // if debug is set but not any other DebugSend* then we will default to sending to debugcerr + + bool mDoRunDebugshow; + + public: + tRunMode getTRunMode() const { return mRunMode; } + bool getDebug() const { return mDebug; } + bool getDebugSendToFile() const { return mDebugSendToFile; } + bool getDebugSendToCerr() const { return mDebugSendToCerr; } + bool getDoRunDebugshow() const { return mDoRunDebugshow; } + + cRunOptions(); + + vector<string> ExecuteRunoptionsAndRemoveThem(const vector<string> & args); + void Exec(const string & runoption); // eg: Exec("+debug"); + + void Normalize(); +}; + +extern cRunOptions gRunOptions; + + +} // namespace nOT + + + +#endif + diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp new file mode 100644 index 000000000..e1d7d9a14 --- /dev/null +++ b/contrib/otshell_utils/utils.cpp @@ -0,0 +1,804 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug + +/* See other files here for the LICENCE that applies here. */ +/* See header file .hpp for info */ + +#include <algorithm> +#include <functional> +#include <cctype> +#include <locale> +#include <fstream> +#include <iostream> +#include <iomanip> + +#include "utils.hpp" + +#include "ccolor.hpp" + +#include "lib_common1.hpp" + +#include "runoptions.hpp" + +#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64) + #define OS_TYPE_WINDOWS +#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__) + #define OS_TYPE_POSIX +#else + #warning "Compiler/OS platform is not recognized. Just assuming it will work as POSIX then" + #define OS_TYPE_POSIX +#endif + +#if defined(OS_TYPE_WINDOWS) + #include <windows.h> +#elif defined(OS_TYPE_POSIX) + #include <sys/types.h> + #include <sys/stat.h> + #include <unistd.h> +#else + #error "Compiler/OS platform detection failed - not supported" +#endif + + +namespace nOT { +namespace nUtils { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +// ==================================================================== + +// Numerical values of the debug levels - see hpp +const int _debug_level_nr_dbg3=20; +const int _debug_level_nr_dbg2=30; +const int _debug_level_nr_dbg1=40; +const int _debug_level_nr_info=50; +const int _debug_level_nr_note=60; +const int _debug_level_nr_fact=75; +const int _debug_level_nr_mark=80; +const int _debug_level_nr_warn=90; +const int _debug_level_nr_erro=100; + +// ==================================================================== + +myexception::myexception(const char * what) + : std::runtime_error(what) +{ } + +myexception::myexception(const std::string &what) + : std::runtime_error(what) +{ } + +void myexception::Report() const { + _erro("Error: " << what()); +} + +//myexception::~myexception() { } + +// ==================================================================== + +// text trimming +// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & ltrim(std::string &s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace)))); + return s; +} + +std::string & rtrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end()); + return s; +} + +std::string & trim(std::string &s) { + return ltrim(rtrim(s)); +} + +std::string get_current_time() { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + time_t time_now = std::chrono::system_clock::to_time_t(now); + std::chrono::high_resolution_clock::duration duration = now.time_since_epoch(); + int64_t micro = std::chrono::duration_cast<std::chrono::microseconds>(duration).count(); + + // std::localtime() - This function may not be thread-safe. + #ifdef OS_TYPE_WINDOWS + struct tm * tm_pointer = std::localtime( &time_now ); // thread-safe on mingw-w64 (thread local variable) and on MSVC btw + // http://stackoverflow.com/questions/18551409/localtime-r-support-on-mingw + // tm_pointer points to thread-local data, memory is owned/managed by the system/library + #else + // linux, freebsd, have this + struct tm tm_object; // automatic storage duration http://en.cppreference.com/w/cpp/language/storage_duration + struct tm * tm_pointer = & tm_object; // just point to our data + auto x = localtime_r( &time_now , tm_pointer ); // modifies our own (this thread) data in tm_object, this is safe http://linux.die.net/man/3/localtime_r + if (x != tm_pointer) return "(internal error in get_current_time)"; // redundant check in case of broken implementation of localtime_r + #endif + // tm_pointer now points to proper time data, and that memory is automatically managed + if (!tm_pointer) return "(internal error in get_current_time - NULL)"; // redundant check in case of broken implementation of used library methods + + std::stringstream stream; + stream << std::setfill('0') + << std::setw(2) << tm_pointer->tm_year+1900 + << '-' << std::setw(2) << tm_pointer->tm_mon+1 + << '-' << std::setw(2) << tm_pointer->tm_mday + << ' ' << std::setw(2) << tm_pointer->tm_hour + << ':' << std::setw(2) << tm_pointer->tm_min + << ':' << std::setw(2) << tm_pointer->tm_sec + << '.' << std::setw(6) << (micro%1000000); // 6 because microseconds + return stream.str(); +} + +cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data) + +std::recursive_mutex gLoggerGuard; // extern +std::atomic<int> gLoggerGuardDepth; // extern + +std::atomic<int> & gLoggerGuardDepth_Get() { + // TODO std::once would be nicer here + + static bool once=0; + + if (!once) { // initialize it once + once=1; + gLoggerGuardDepth=0; + } + + return gLoggerGuardDepth; // global, atomic counter +} + + +// ==================================================================== + +namespace nDetail { + +const char* DbgShortenCodeFileName(const char *s) { + const char *p = s; + const char *a = s; + + bool inc=1; + while (*p) { + ++p; + if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash + if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character) + } + return a; +} + +} + +// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795 +#ifndef _MSC_VER +template<typename T, typename ...Args> +std::unique_ptr<T> make_unique( Args&& ...args ) +{ + return std::unique_ptr<T>( new T( std::forward<Args>(args)... ) ); +} +#else + using std::make_unique; +#endif +// ==================================================================== + +char cFilesystemUtils::GetDirSeparatorSys() { + // TODO nicer os detection? + #if defined(OS_TYPE_POSIX) + return '/'; + #elif defined(OS_TYPE_WINDOWS) + return '\\'; + #else + #error "Do not know how to compile this for your platform." + #endif +} + +char cFilesystemUtils::GetDirSeparatorInter() { + return '/'; +} + +string cFilesystemUtils::FileInternalToSystem(const std::string &name) { + string ret; + ret.resize(name.size()); + std::replace_copy(name.begin(), name.end(), ret.begin(), + GetDirSeparatorInter() , GetDirSeparatorSys()); + return ret; +} + +string cFilesystemUtils::FileSystemToInternal(const std::string &name) { + string ret; + ret.reserve(name.size()); + std::replace_copy(name.begin(), name.end(), ret.begin(), + GetDirSeparatorSys() , GetDirSeparatorInter()); + return ret; +} + +bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) { + const bool dbg=false; + //struct stat st; + const char dirchS = cFilesystemUtils::GetDirSeparatorSys(); + const char dirchI = cFilesystemUtils::GetDirSeparatorInter(); + std::istringstream iss(dir); + string partI; // current par is in internal format (though it should not matter since it doesn't contain any slashes). eg "bar" + string sofarS=""; // sofarS - the so far created dir part is in SYSTEM format. eg "foo/bar" + if (dir.size()<1) return false; // illegal name + // dir[0] is valid from here + if ( only_below && ((dir[0]==dirchS) || (dir[0]==dirchI))) return false; // no jumping to top (on any os) + + while (getline(iss,partI,dirchI)) { // get new component eg "bar" into part + if (dbg) cout << '['<<partI<<']' << endl; + sofarS += partI; + if (partI.size()<1) return false; // bad format? + if ((only_below) && (partI=="..")) return false; // trying to go up + + if (dbg) cout << "test ["<<sofarS<<"]"<<endl; + // TODO nicer os detection? + #if defined(OS_TYPE_POSIX) + struct stat st; + bool exists = stat(sofarS.c_str() ,&st) == 0; // * + if (exists) { + if (! S_ISDIR(st.st_mode)) { + // std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl; + return false; // exists but is a file nor dir + } + } + #elif defined(OS_TYPE_WINDOWS) + DWORD dwAttrib = GetFileAttributesA(sofarS.c_str()); + bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)); + #else + #error "Do not know how to compile this for your platform." + #endif + + if (!exists) { + if (dbg) cout << "mkdir ["<<sofarS<<"]"<<endl; + #if defined(OS_TYPE_POSIX) + bool ok = 0== mkdir(sofarS.c_str(), 0700); // *** + #elif defined(OS_TYPE_WINDOWS) + bool ok = (bool) CreateDirectoryA(sofarS.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16 + #else + #error "Do not know how to compile this for your platform." + #endif + if (!ok) return false; + } + sofarS += dirchS; + } + return true; +} +// ==================================================================== + +namespace nDetail { + +struct channel_use_info { ///< feedback information about using (e.g. opening) given debug channel - used internally by logging system +/// TODO not yet used in code +/// e.g. used to write into channel net/in/all that given message was a first logged message from never-before-logged thread or PID etc + bool m_was_interesting; ///< anything interesting happened when using the channel? + std::vector<std::string> m_extra_msg; ///< any additional messages about this channel use +}; + +cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) { +} + +cDebugScopeGuard::~cDebugScopeGuard() { + if (mLevel != -1) { + gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush; + } +} + +void cDebugScopeGuard::Assign(const string &chan, const int level, const string &msg) { + mChan=chan; + mLevel=level; + mMsg=msg; +} + +} // namespace nDetail + +// ==================================================================== + +cLogger::cLogger() : +mStream(NULL), +mStreamBrokenDebug(NULL), +mIsBroken(true), // before constructor finishes +mLevel(_debug_level_nr_warn), +mThread2Number_Biggest(0), // the CURRENT biggest value (no thread yet in map) +mPid2Number_Biggest(0) +{ + mStream = & std::cout; + mStreamBrokenDebug = & std::cerr; // the backup stream + *mStreamBrokenDebug << "Creating the logger system" << endl; + mIsBroken=false; // ok, constr. succeeded, so string is not broken now + + // this is here, because it could be using logging itself to log creation of first thread/PID etc + Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1 + Pid2Number( getpid() ); // add this proces ID as first one +} + +cLogger::~cLogger() { + for (auto pair : mChannels) { + std::ofstream *ptr = pair.second; + delete ptr; + pair.second=NULL; + } +} + +void cLogger::SetStreamBroken() { + SetStreamBroken("(no additional details about this problem)"); +} + +void cLogger::SetStreamBroken(const std::string &msg) { + _dbg_dbg("Stream is broken (msg: " << msg << ")"); + if (!mIsBroken) { // if not already marked as broken + _dbg_dbg("(It was not broken before)"); + std::cerr << OT_CODE_STAMP << "WARNING: due to a problem in the debug/logging system itself ("<<msg<<") - we are switching back to fallback stream (e.g. cerr)" << std::endl; + if (mStreamBrokenDebug == nullptr) { + std::cerr << OT_CODE_STAMP << " ERROR: in addition, while reporting this problem, mStreamBrokenDebug stream is NULL: " << mStreamBrokenDebug << std::endl; + } else { + (*mStreamBrokenDebug) << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<<msg<<") - switching back to fallback stream (e.g. cerr)" << std::endl; + } + mIsBroken = true; + } +} + +std::ostream & cLogger::write_stream(int level) { + return write_stream(level,""); +} + +std::ostream & cLogger::write_stream(int level, const std::string & channel ) { + _dbg_dbg("level="<<level<<" channel="<<channel); + if (level >= mLevel) { + if (mStream) { // TODO now disabling mStream also disables writting to any channel + _dbg_dbg("Selecting output..."); + ostream & output = SelectOutput(level,channel); + _dbg_dbg("Selecting output... done, output=" << (void*)(&output)); + #if defined(OS_TYPE_WINDOWS) + output << windows_stream(level); + #endif + output << icon(level) << ' '; + std::thread::id this_id = std::this_thread::get_id(); + output << "{" << Thread2Number(this_id) << "}"; + auto nicePid = Pid2Number(getpid()); + if (nicePid>0) output << " {p" << nicePid << "}"; + output << ' '; + return output; // <--- return + } else _dbg_dbg("Not writting: No mStream"); + } else _dbg_dbg("Not writting: Too low level level="<<level<<" not >= mLevel="<<mLevel); + return g_nullstream; +} + +std::string cLogger::GetLogBaseDir() const { + return "log"; +} + +void cLogger::OpenNewChannel(const std::string & channel) noexcept { + try { + _dbg_dbg("Openning channel for channel="<<channel); + OpenNewChannel_(channel); + } + catch (const std::exception &except) { + SetStreamBroken(OT_CODE_STAMP + " Got exception when opening debug channel: " + ToStr(except.what())); + } + catch (...) { + SetStreamBroken(OT_CODE_STAMP + " Got not-standard exception when opening debug channel."); + } +} + +void cLogger::OpenNewChannel_(const std::string & channel) { // channel=="net/sleep" + _dbg_dbg("Openning channel for channel="<<channel); + size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparatorInter()); + + string fname_system; // the full file name in system format + + if (last_split==string::npos) { // The channel name has no directory, eg channel=="test" + string dir = GetLogBaseDir(); + string basefile = channel + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile; + fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <- + } + else { // there is a directory eg channel=="net/sleep" + // net/sleep + // ^----- last_split + string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparatorInter() + channel.substr(0, last_split); + string basefile = channel.substr(last_split+1) + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile; + fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <- + bool dirok = cFilesystemUtils::CreateDirTree(dir); + if (!dirok) { string err = "In logger failed to open directory (" + dir +") for channel (" + channel +")"; throw std::runtime_error(err); } + } + + _dbg_dbg("Openning fname_system="<<fname_system); + std::ofstream * thefile = new std::ofstream( fname_system.c_str() ); // file system + *thefile << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl; +// cerr << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl; + _dbg_dbg( "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" ); + mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) ); // <- created the channel mapping +} + +std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept { + try { + if (mIsBroken) { + _dbg_dbg("The stream is broken mIsBroken="<<mIsBroken<<" so will return backup stream"); + return *mStreamBrokenDebug; + } + if (channel=="") { + _dbg_dbg("No channel given (channel="<<channel<<") so will return main stream"); + return *mStream; + } + + auto obj = mChannels.find(channel); + if (obj == mChannels.end()) { // not found - need to make new channel + _dbg_dbg("No stream openened for channel="<<channel<<" so will create it now"); + OpenNewChannel(channel); // <- create channel + obj = mChannels.find(channel); // find again + if (obj == mChannels.end()) { // still not found! something is wrong + SetStreamBroken( OT_CODE_STAMP + " WARNING: can not get stream for channel="+ToStr(channel)+" level="+ToStr(channel) ); + return *mStreamBrokenDebug; + } + } + auto the_stream_ptr = obj->second; + _dbg_dbg("Found the stream file for channel="<<channel<<" as the_stream_ptr="<<the_stream_ptr); + ASRT(the_stream_ptr); + return *the_stream_ptr; // <--- RETURN + } + catch (std::exception &except) { + SetStreamBroken( OT_CODE_STAMP + " Got exception: " + ToStr(except.what()) ); + _dbg_dbg("Exception! Returning broken stream"); + return *mStreamBrokenDebug; + } + catch (...) { + SetStreamBroken( OT_CODE_STAMP + " Got not-standard exception."); + _dbg_dbg("Exception! Returning broken stream"); + return *mStreamBrokenDebug; + } + + // dead code +} + +void cLogger::setOutStreamFile(const string &fname) { // switch to using this file + _mark("WILL SWITCH DEBUG NOW to file: " << fname); + mOutfile = make_unique<std::ofstream>(fname); + mStream = & (*mOutfile); + _mark("Started new debug, to file: " << fname); +} + +void cLogger::setOutStreamFromGlobalOptions() { + if ( gRunOptions.getDebug() ) { + if ( gRunOptions.getDebugSendToFile() ) { + mOutfile = make_unique<std::ofstream> ("debuglog.txt"); + mStream = & (*mOutfile); + } + else if ( gRunOptions.getDebugSendToCerr() ) { + mStream = & std::cerr; + } + else { + mStream = & g_nullstream; + } + } + else { + mStream = & g_nullstream; + } +} + +void cLogger::setDebugLevel(int level) { + bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level) + if (note_before) _note("Setting debug level to "<<level); + mLevel = level; + if (!note_before) _note("Setting debug level to "<<level); +} + +std::string cLogger::icon(int level) const { + // TODO replan to avoid needles converting back and forth char*, string etc + + using namespace zkr; + #if defined(OS_TYPE_POSIX) + if (level >= 100) return cc::back::lightred + ToStr(cc::fore::lightyellow) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ; + if (level >= 90) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("Warn ") + ToStr(cc::fore::red)+ " " ; + if (level >= 80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " "; + if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " "; + if (level >= 70) return cc::fore::green + ToStr("Note "); + if (level >= 50) return cc::fore::cyan + ToStr("info "); + if (level >= 40) return cc::fore::lightwhite + ToStr("dbg "); + if (level >= 30) return cc::fore::lightblue + ToStr("dbg "); + if (level >= 20) return cc::fore::blue + ToStr("dbg "); + + #elif defined(OS_TYPE_WINDOWS) + if (level >= 100) return ToStr("ERROR "); + if (level >= 90) return ToStr("Warn "); + if (level >= 80) return ToStr("MARK "); + if (level >= 75) return ToStr("FACT "); + if (level >= 70) return ToStr("Note "); + if (level >= 50) return ToStr("info "); + if (level >= 40) return ToStr("dbg "); + if (level >= 30) return ToStr("dbg "); + if (level >= 20) return ToStr("dbg "); + #endif + + return " "; +} + +std::string cLogger::endline() const { + #if defined(OS_TYPE_POSIX) + return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc + #elif defined(OS_TYPE_WINDOWS) + return ToStr("\n"); + #endif +} + +int cLogger::Thread2Number(const std::thread::id id) { + auto found = mThread2Number.find( id ); + if (found == mThread2Number.end()) { // new one + mThread2Number_Biggest++; + mThread2Number[id] = mThread2Number_Biggest; + _info_c("dbg/main", "This is a new thread (used in debug), thread id="<<id); // can cause some recursion + return mThread2Number_Biggest; + } else { + return mThread2Number[id]; + } +} + +int cLogger::Pid2Number(const t_anypid id) { + auto found = mPid2Number.find( id ); + if (found == mPid2Number.end()) { // new one + mPid2Number_Biggest++; + mPid2Number[id] = mPid2Number_Biggest; + _info_c("dbg/main", "This is a new process (used in debug), process pid="<<id); // can cause some recursion + return mPid2Number_Biggest; + } else { + return mPid2Number[id]; + } +} + +// ==================================================================== +// object gCurrentLogger is defined later - in global namespace below + + +// ==================================================================== +// vector debug + +void DisplayStringEndl(std::ostream & out, const std::string text) { + out << text; + out << std::endl; +} + +std::string SpaceFromEscape(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] == '\\' && s[i+1] ==32) + newStr<<""; + else + newStr<<s[i]; + } + return newStr.str(); +} + +std::string EscapeFromSpace(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] == 32) + newStr << "\\" << " "; + else + newStr << s[i]; + } + return newStr.str(); +} + + +std::string EscapeString(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] >=32 && s[i] <= 126) + newStr<<s[i]; + else + newStr<<"\\"<< (int) s[i]; + } + + return newStr.str(); +} + + +bool CheckIfBegins(const std::string & beggining, const std::string & all) { + if (all.compare(0, beggining.length(), beggining) == 0) { + return 1; + } + else { + return 0; + } +} + +bool CheckIfEnds (std::string const & ending, std::string const & all){ + if (all.length() >= ending.length()) { + return (0 == all.compare (all.length() - ending.length(), ending.length(), ending)); + } else { + return false; + } +} + + +vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib) { + vector<string> ret; + for ( auto rec : possib) { // check of possibilities + if (CheckIfBegins(sofar,rec)) { + rec = EscapeFromSpace(rec); + ret.push_back(rec); // this record matches + } + } + return ret; +} + +char GetLastChar(const std::string & str) { // TODO unicode? + auto s = str.length(); + if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP); + return str.at( s - 1); +} + +std::string GetLastCharIf(const std::string & str) { // TODO unicode? + auto s = str.length(); + if (s==0) return ""; // empty string signalizes ther is nothing to be returned + return std::string( 1 , str.at( s - 1) ); +} + +// ==================================================================== + +// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no. +// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :) + +void Assert(bool result, const std::string &stamp, const std::string &condition) { + if (!result) { + _erro("Assert failed at "+stamp+": ASSERT( " << condition << ")"); + throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")"); + } +} + +// ==================================================================== +// advanced string + +const std::string GetMultiline(string endLine) { + std::string result(""); // Taken from OT_CLI_ReadUntilEOF + while (true) { + std::string input_line(""); + if (std::getline(std::cin, input_line, '\n')) + { + input_line += "\n"; + if (input_line[0] == '~') + break; + result += input_line; + } + if (std::cin.eof() ) + { + std::cin.clear(); + break; + } + if (std::cin.fail() ) + { + std::cin.clear(); + break; + } + if (std::cin.bad()) + { + std::cin.clear(); + break; + } + } + return result; +} + +vector<string> SplitString(const string & str){ + std::istringstream iss(str); + vector<string> vec { std::istream_iterator<string>{iss}, std::istream_iterator<string>{} }; + return vec; +} + +bool checkPrefix(const string & str, char prefix) { + if (str.at(0) == prefix) + return true; + return false; +} + +// ==================================================================== +// operation on files + + +#ifdef __unix + +void cEnvUtils::GetTmpTextFile() { + // TODO make this name configurable (depending on project) + char filename[] = "/tmp/otshellutils_text.XXXXXX"; + fd = mkstemp(filename); + if (fd == -1) { + _erro("Can't create the file: " << filename); + return; + } + mFilename = filename; +} + +void cEnvUtils::CloseFile() { + close(fd); + unlink( mFilename.c_str() ); +} + +void cEnvUtils::OpenEditor() { + char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file + if (editor == NULL) + editor = std::getenv("VISUAL"); + if (editor == NULL) + editor = std::getenv("EDITOR"); + + string command; + if (editor != NULL) + command = ToStr(editor) + " " + mFilename; + else + command = "/usr/bin/editor " + mFilename; + _dbg3("Opening editor with command: " << command); + if ( system( command.c_str() ) == -1 ) + _erro("Cannot execute system command: " << command); +} + +const string cEnvUtils::ReadFromTmpFile() { + std::ifstream ifs(mFilename); + string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>()); + return msg; +} + +const string cEnvUtils::Compose() { + GetTmpTextFile(); + OpenEditor(); + string input = ReadFromTmpFile(); + CloseFile(); + return input; +} + +#endif + +const string cEnvUtils::ReadFromFile(const string path) { + std::ifstream ifs(path); + string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>()); + return msg; +} + +void hintingToTxt(std::fstream & file, string command, vector<string> &commands) { + if(file.good()) { + file<<command<<"~"<<endl; + for (auto a: commands) { + file <<a<< " "; + file.flush(); + } + file<<endl; + } +} + +string stringToColor(const string &hash) { + // Generete vector with all possible light colors + vector <string> lightColors; + using namespace zkr; + lightColors.push_back(cc::fore::lightblue); + lightColors.push_back(cc::fore::lightred); + lightColors.push_back(cc::fore::lightmagenta); + lightColors.push_back(cc::fore::lightgreen); + lightColors.push_back(cc::fore::lightcyan); + lightColors.push_back(cc::fore::lightyellow); + lightColors.push_back(cc::fore::lightwhite); + + int sum=0; + + for (auto ch : hash) sum+=ch; + auto color = sum%(lightColors.size()-1); + + return lightColors.at( color ); +} + + +// ==================================================================== +// algorthms + + +} // namespace nUtil + + +} // namespace OT + +// global namespace + +const extern int _dbg_ignore = 0; // see description in .hpp + +std::string GetObjectName() { + //static std::string * name=nullptr; + //if (!name) name = new std::string("(global)"); + return ""; +} + +// ==================================================================== + +nOT::nUtils::cLogger gCurrentLogger; + diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp new file mode 100644 index 000000000..297f13ac0 --- /dev/null +++ b/contrib/otshell_utils/utils.hpp @@ -0,0 +1,532 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug + +/* See other files here for the LICENCE that applies here. */ + +#include "ccolor.hpp" +#ifndef INCLUDE_OT_NEWCLI_UTILS +#define INCLUDE_OT_NEWCLI_UTILS + +#include "lib_common1.hpp" +#ifdef __unix + #include <unistd.h> +#endif + +#if defined(_WIN32) + #include"windows_stream.h" +#endif + +#ifndef CFG_WITH_TERMCOLORS + //#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)." +#endif + +///Macros related to automatic deduction of class name etc; +#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } +#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public: + +// define this to debug the debug system itself: +// #define opt_debug_debug + +#ifdef opt_debug_debug + #define _dbg_dbg(X) do { std::cerr<<"_dbg_dbg: " << OT_CODE_STAMP << " {thread=" << std::this_thread::get_id()<<"} " \ + << " {pid="<<getpid()<<"} " << ": " << X << std::endl; } while(0) +#else + #define _dbg_dbg(X) do { } while(0) +#endif + +namespace nOT { + +namespace nUtils { + +/// @brief general based for my runtime errors +class myexception : public std::runtime_error { + public: + myexception(const char * what); + myexception(const std::string &what); + //virtual ~myexception(); + virtual void Report() const; +}; + +/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc. +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +// ====================================================================================== +/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again +/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & ltrim(std::string &s); ///< left trim +std::string & rtrim(std::string &s); ///< right trim + +// ====================================================================================== + +std::string get_current_time(); + +// string conversions +template <class T> +std::string ToStr(const T & obj) { + std::ostringstream oss; + oss << obj; + return oss.str(); +} + +struct cNullstream : std::ostream { + cNullstream() : std::ios(0), std::ostream(0) {} +}; +extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data) +// ========== debug ========== +// _dbg_ignore is moved to global namespace (on purpose) + +// TODO make _dbg_ignore thread-safe everywhere + +extern std::recursive_mutex gLoggerGuard; // the mutex guarding logging/debugging code e.g. protecting streams, files, etc + +std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton of counter (it guarantees initializing it to 0). This counter shows the current recursion (re-entrant) level of debug macros. + +// TODO more debug of the debug system: +// detect lock() error e.g. recursive limit +// detect stream e.g. operator<< error + +#define _debug_level(LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + _dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" VAR: " << VAR ); \ + auto level=LEVEL; short int part=0; \ + try { \ + std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ + part=1; \ + try { \ + ++nOT::nUtils::gLoggerGuardDepth_Get(); \ +/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \ + gCurrentLogger.write_stream(LEVEL,"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ + part=9; \ + } catch(...) { \ + gCurrentLogger.write_stream(std::max(level,90),"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \ + --nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \ + } \ + --nOT::nUtils::gLoggerGuardDepth_Get(); \ + } catch(...) { if (part<8) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline(); throw ; } \ + } } while(0) + +// info for code below: oss object is normal stack variable, using it does not need lock protection +#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + _dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" CHANNEL="<<CHANNEL<<" VAR: " << VAR ); \ + auto level=LEVEL; short int part=0; \ + try { \ + std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ + part=1; \ + try { \ + ++nOT::nUtils::gLoggerGuardDepth_Get(); \ + std::ostringstream oss; \ + oss << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ + std::string as_string = oss.str(); \ + _dbg_dbg("START will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \ +/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \ + gCurrentLogger.write_stream(LEVEL,"" ) << as_string << gCurrentLogger.endline() << std::flush; \ + gCurrentLogger.write_stream(LEVEL,CHANNEL) << as_string << gCurrentLogger.endline() << std::flush; \ + _dbg_dbg("DONE will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \ + part=9; \ + } catch(...) { \ + gCurrentLogger.write_stream(std::max(level,90),CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \ + --nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \ + } \ + --nOT::nUtils::gLoggerGuardDepth_Get(); \ + } catch(...) { if (part<8) gCurrentLogger.write_stream(100,CHANNEL)<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline(); throw ; } \ + } } while(0) + +// Numerical values of the debug levels - are defined here as const ints. Full name (with namespace) given for clarity. +extern const int _debug_level_nr_dbg3; +extern const int _debug_level_nr_dbg2; +extern const int _debug_level_nr_dbg1; +extern const int _debug_level_nr_info; +extern const int _debug_level_nr_note; +extern const int _debug_level_nr_fact; +extern const int _debug_level_nr_mark; +extern const int _debug_level_nr_warn; +extern const int _debug_level_nr_erro; + +#define _dbg3(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg3,VAR) // details - most detailed +#define _dbg2(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg2,VAR) // details - a bit more important +#define _dbg1(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg1,VAR) // details - more important +#define _info(VAR) _debug_level( nOT::nUtils::_debug_level_nr_info,VAR) // information +#define _note(VAR) _debug_level( nOT::nUtils::_debug_level_nr_note,VAR) // more interesting information +#define _fact(VAR) _debug_level( nOT::nUtils::_debug_level_nr_fact,VAR) // interesting events that could be interesting even for user, for logical/business things +#define _mark(VAR) _debug_level( nOT::nUtils::_debug_level_nr_mark,VAR) // marked actions +#define _warn(VAR) _debug_level( nOT::nUtils::_debug_level_nr_warn,VAR) // some problems +#define _erro(VAR) _debug_level( nOT::nUtils::_debug_level_nr_erro,VAR) // errors + +#define _dbg3_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg3, VAR) // details - most detailed +#define _dbg2_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg2, VAR) // details - a bit more important +#define _dbg1_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg1, VAR) // details - more important +#define _info_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_info, VAR) // information +#define _note_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_note, VAR) // more interesting information +#define _fact_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_fact, VAR) // interesting events that could be interesting even for user, for logical/business things +#define _mark_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_mark, VAR) // marked actions +#define _warn_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_warn, VAR) // some problems +#define _erro_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_erro, VAR) // errors + +// lock // because of VAR +#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \ + std::ostringstream debug_detail_oss; \ + nOT::nUtils::gLoggerGuard.lock(); \ + debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \ + nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \ + if (_dbg_ignore<LEVEL) debugScopeGuard.Assign(CHANNEL,LEVEL, debug_detail_oss.str()); \ + if (_dbg_ignore<LEVEL) _debug_level_c(CHANNEL,LEVEL,debug_detail_oss.str() + " ... begin"); \ + nOT::nUtils::gLoggerGuard.unlock(); +#define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR) + +#define _scope_dbg1(VAR) _scope_debug_level( _debug_level_nr_dbg3, VAR) +#define _scope_dbg2(VAR) _scope_debug_level( _debug_level_nr_dbg2, VAR) +#define _scope_dbg3(VAR) _scope_debug_level( _debug_level_nr_dbg1, VAR) +#define _scope_info(VAR) _scope_debug_level( _debug_level_nr_info, VAR) +#define _scope_note(VAR) _scope_debug_level( _debug_level_nr_note, VAR) +#define _scope_fact(VAR) _scope_debug_level( _debug_level_nr_fact, VAR) +#define _scope_mark(VAR) _scope_debug_level( _debug_level_nr_mark, VAR) +#define _scope_warn(VAR) _scope_debug_level( _debug_level_nr_warn, VAR) +#define _scope_erro(VAR) _scope_debug_level( _debug_level_nr_erro, VAR) + +/*** +@brief do not use this namespace directly, it is implementation detail. +*/ +namespace nDetail { + +/*** +@brief a Debug scope-guard, to log a debug message when current scope is left. Do NOT use this directly, +only use it via the macros like _scope_dbg1 etc. +*/ +class cDebugScopeGuard { + protected: + string mMsg; + int mLevel; + string mChan; + public: + cDebugScopeGuard(); + ~cDebugScopeGuard(); + void Assign(const string &chan, const int level, const string &msg); +}; + +const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug + +} // namespace nDetail + +// ========== logger ========== + +namespace nDetail { + struct channel_use_info; +} // namespace nDetail + +/*** +@brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly! +@author rfree (maintainer) +@thread this class is NOT thread safe and must used only by one thread at once (use it via ot_debug_macros like _info macro they do proper locking) +*/ +class cLogger { + public: + cLogger(); + ~cLogger(); + std::ostream & write_stream(int level); ///< starts a new message on given level (e.g. writes out the icon/tag) and returns stream to output to + std::ostream & write_stream(int level, const std::string & channel); ///< the same but with name of the debug channel + + void setOutStreamFromGlobalOptions(); // set debug level, file etc - according to global Options + void setOutStreamFile(const std::string &fname); // switch to using this file + void setDebugLevel(int level); // change the debug level e.g. to mute debug from now + + std::string icon(int level) const; ///< returns "icon" for given debug level. It is text, might include color controll characters + std::string endline() const; ///< returns string to be written at end of message + + protected: + typedef long int t_anypid; // a portable representation of PID. long int should cover all platforms + + void SetStreamBroken(); ///< call in case of internal error in logger (e.g. can not open a file) + void SetStreamBroken(const std::string &msg); ///< same but with error message + + unique_ptr<std::ofstream> mOutfile; + std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream + std::ostream * mStreamBrokenDebug; ///< pointing only! this is a pointer to some stream that should be used when normal debugging is broken eg std::cerr + bool mIsBroken; ///< is the debugging system broken (this should be set when internal problems occur and should cause fallback to std::cerr) + + std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class + + int mLevel; ///< current debug level + + std::ostream & SelectOutput(int level, const std::string & channel) noexcept; ///< returns a proper stream for this level and channel (always usable string) + void OpenNewChannel(const std::string & channel) noexcept; ///< tries to prepare this channel. does NOT guarantee to created mChannels[] entry! + void OpenNewChannel_(const std::string & channel); ///< internal function, will throw in case of problems + std::string GetLogBaseDir() const; + + std::map< std::thread::id , int > mThread2Number; ///< change long thread IDs into a short nice number to show + int mThread2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically + int Thread2Number(const std::thread::id id); ///< convert the system's thread id into a nice short our id; make one if new thread + + std::map< t_anypid , int > mPid2Number; ///< change long proces PID into a short nice number to show + int mPid2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically + int Pid2Number(const t_anypid id); ///< convert the system's PID id into a nice short our id; make one if new thread +}; + + + +// ==================================================================== +// vector debug + +template <class T> +std::string vectorToStr(const T & v) { + std::ostringstream oss; + for(auto rec: v) { + oss << rec <<","; + } + return oss.str(); +} + +template <class T> +void DisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") { + std::copy( v.begin(), v.end(), std::ostream_iterator<T>(out, delim.c_str()) ); +} + +template <class T> +void EndlDisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") { + out << std::endl; + DisplayVector(out,v,delim); +} + +template <class T> +void DisplayVectorEndl(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") { + DisplayVector(out,v,delim); + out << std::endl; +} + +template <class T> +void DbgDisplayVector(const std::vector<T> &v, const std::string &delim=" ") { + std::cerr << "["; + std::copy( v.begin(), v.end(), std::ostream_iterator<T>(std::cerr, delim.c_str()) ); + std::cerr << "]"; +} + +string stringToColor(const string &hash); +template <class T, class T2> +void DisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") { + auto *no_color = zkr::cc::fore::console; + for(auto var : m) { + out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl; + } + +} + +template <class T, class T2> +void EndlDisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") { + out << endl; + for(auto var : m) { + out << var.first << delim << var.second << endl; + } +} + +template <class T, class T2> +void DbgDisplayMap(const std::map<T, T2> &m, const std::string &delim=" ") { + for(auto var : m) { + std::cerr << var.first << delim << var.second << endl; + } +} + + +template <class T> +void DbgDisplayVectorEndl(const std::vector<T> &v, const std::string &delim=" ") { + DbgDisplayVector(v,delim); + std::cerr << std::endl; +} + +void DisplayStringEndl(std::ostream & out, const std::string text); + +bool CheckIfBegins(const std::string & beggining, const std::string & all); +bool CheckIfEnds (std::string const & ending, std::string const & all); +std::string SpaceFromEscape(const std::string &s); +std::string EscapeFromSpace(const std::string &s); +vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib); +char GetLastChar(const std::string & str); +std::string GetLastCharIf(const std::string & str); // TODO unicode? +std::string EscapeString(const std::string &s); + + +template <class T> +std::string DbgVector(const std::vector<T> &v, const std::string &delim="|") { + std::ostringstream oss; + oss << "["; + bool first=true; + for(auto vElement : v) { if (!first) oss<<delim; first=false; oss <<vElement ; } + oss << "]"; + //std::copy( v.begin(), v.end(), std::ostream_iterator<T>(oss, delim.c_str()) ); + return oss.str(); +} + +template <class T> +std::ostream & operator<<(std::ostream & os, const map< T, vector<T> > & obj){ + os << "["; + for(auto const & elem : obj) { + os << " [" << elem.first << "=" << DbgVector(elem.second) << "] "; + } + os << "]"; + return os; +} + +template <class T, class T2> +std::string DbgMap(const map<T, T2> & map) { + std::ostringstream oss; + oss << map; + return oss.str(); +} + +// ==================================================================== +// assert + +// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no. +// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :) +#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0) + +void Assert(bool result, const std::string &stamp, const std::string &condition); + +// ==================================================================== +// advanced string + +const std::string GetMultiline(string endLine = "~"); +vector<string> SplitString(const string & str); + +bool checkPrefix(const string & str, char prefix = '^'); + +// ==================================================================== +// nUse utils + +enum class eSubjectType {Account, Asset, User, Server, Unknown}; + +string SubjectType2String(const eSubjectType & type); +eSubjectType String2SubjectType(const string & type); + +// ==================================================================== +// operation on files + +/// @brief tools related to filesystem +/// @author rfree (maintainer) +class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later) + public: + static bool CreateDirTree(const std::string & dir, bool only_below=false); + static char GetDirSeparatorSys(); /// < eg '/' or '\' + static char GetDirSeparatorInter(); /// < internal is '/' + static string FileInternalToSystem(const std::string &name); ///< converts from internal file name string to system file name string + static string FileSystemToInternal(const std::string &name); ///< converts from system file name string to internal file name string +}; + + +/// @brief utils to e.g. edit a file from console +/// @author rfree (maintainer) +class cEnvUtils { + int fd; + string mFilename; + + void GetTmpTextFile(); + void CloseFile(); + void OpenEditor(); + const string ReadFromTmpFile(); +public: + const string Compose(); + const string ReadFromFile(const string path); +}; +void hintingToTxt(std::fstream & file, string command, vector<string> &commands); +void generateQuestions (std::fstream & file, string command); +void generateAnswers (std::fstream & file, string command, vector<string> &completions); + +// ==================================================================== + +namespace nOper { // nOT::nUtils::nOper +// cool shortcut operators, like vector + vecotr operator working same as string (appending) +// isolated to namespace because it's unorthodox ide to implement this + +using namespace std; + +// TODO use && and move? +template <class T> +vector<T> operator+(const vector<T> &a, const vector<T> &b) { + vector<T> ret = a; + ret.insert( ret.end() , b.begin(), b.end() ); + return ret; +} + +template <class T> +vector<T> operator+(const T &a, const vector<T> &b) { + vector<T> ret(1,a); + ret.insert( ret.end() , b.begin(), b.end() ); + return ret; +} + +template <class T> +vector<T> operator+(const vector<T> &a, const T &b) { + vector<T> b_vector(1,a); + return a + b_vector; +} + +template <class T> +vector<T>& operator+=(vector<T> &a, const vector<T> &b) { + a.insert( a.end() , b.begin(), b.end() ); + return a; +} + +// map +template <class TK,class TV> +map<TK,TV> operator+(const map<TK,TV> &a, const map<TK,TV> &b) { + map<TK,TV> ret = a; + for (const auto & elem : b) { + ret.insert(elem); + } + return ret; +} + + +} // nOT::nUtils::nOper + +// ==================================================================== + +// ==================================================================== + +// Algorithms + +// ==================================================================== +// ==================================================================== + + +/** +@brief Special type that on creation will be initialized to have value INIT given as template argument. +Might be usefull e.g. to express in the declaration of class what will be the default value of member variable +See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm +Probably not needed when using boost in your project. +*/ +template <class T, T INIT> +class value_init { + private: + T data; + public: + value_init(); + + T& operator=(const T& v) { data=v; return *this; } + operator T const &() const { return data; } + operator T&() { return data; } +}; + +template <class T, T INIT> +value_init<T, INIT>::value_init() : data(INIT) { } + +} // namespace nUtils + +} // namespace nOT + + +// global namespace +extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc + +std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently) + +const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks, +// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there. +// Or to make it runtime by providing a class normal member and editing it in runtime + +#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]")) + + + + +#endif + diff --git a/contrib/otshell_utils/windows_stream.cpp b/contrib/otshell_utils/windows_stream.cpp new file mode 100644 index 000000000..59d8b12a3 --- /dev/null +++ b/contrib/otshell_utils/windows_stream.cpp @@ -0,0 +1,64 @@ +#if defined(_WIN32)
+#include "windows_stream.h"
+#include <windows.h>
+
+windows_stream::windows_stream(unsigned int pLevel)
+ :
+ mLevel(pLevel)
+{
+}
+
+std::ostream& operator << (std::ostream &stream, windows_stream const& object)
+{
+ HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+
+ if (object.mLevel >= 100)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_RED | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 90)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 80)
+ {
+ SetConsoleTextAttribute(h_stdout, BACKGROUND_BLUE | BACKGROUND_RED | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 75)
+ {
+ SetConsoleTextAttribute(h_stdout, BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 70)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 50)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 40)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_RED | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 30)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 20)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE);
+ return stream;
+ }
+
+ return stream;
+}
+
+#endif
diff --git a/contrib/otshell_utils/windows_stream.h b/contrib/otshell_utils/windows_stream.h new file mode 100644 index 000000000..859e7ee50 --- /dev/null +++ b/contrib/otshell_utils/windows_stream.h @@ -0,0 +1,20 @@ +#ifndef WINDOWS_STREAM_H
+#define WINDOWS_STREAM_H
+
+#if defined(_WIN32)
+
+#include <string>
+#include <iostream>
+
+class windows_stream
+{
+public:
+ windows_stream(unsigned int pLevel);
+ friend std::ostream& operator<<(std::ostream &stream, windows_stream const& object);
+private:
+ unsigned int mLevel = 0;
+};
+
+#endif // _WIN32
+
+#endif // WINDOWS_STREAM_H
|