diff options
61 files changed, 983 insertions, 411 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6ed13e62..ca6589424 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ jobs: submodules: recursive - uses: actions/cache@v2 with: - path: ~/.ccache + path: /Users/runner/Library/Caches/ccache key: ccache-macos-build-${{ github.sha }} restore-keys: ccache-macos-build- - name: install dependencies diff --git a/.gitmodules b/.gitmodules index 9dacf534f..3cf831bcd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,7 @@ branch = monero [submodule "external/miniupnp"] path = external/miniupnp - url = https://github.com/monero-project/miniupnp - branch = monero + url = https://github.com/miniupnp/miniupnp [submodule "external/rapidjson"] path = external/rapidjson url = https://github.com/Tencent/rapidjson diff --git a/CMakeLists.txt b/CMakeLists.txt index c6d102ba7..d67ed7b9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -714,9 +714,10 @@ else() endif() # linker - if (NOT SANITIZE AND NOT OSSFUZZ AND NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1))) + if (NOT SANITIZE AND NOT OSSFUZZ AND NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND (CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1 OR NOT STATIC)))) # PIE executables randomly crash at startup with ASAN # Windows binaries die on startup with PIE when compiled with GCC <9.x + # Windows dynamically-linked binaries die on startup with PIE regardless of GCC version add_linker_flag_if_supported(-pie LD_SECURITY_FLAGS) endif() add_linker_flag_if_supported(-Wl,-z,relro LD_SECURITY_FLAGS) @@ -68,7 +68,7 @@ OUTPUT_DIRECTORY = doc # performance problems for the file system. # The default value is: NO. -CREATE_SUBDIRS = NO +CREATE_SUBDIRS = YES # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII @@ -754,7 +754,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = src +INPUT = . # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -1552,7 +1552,7 @@ EXTRA_SEARCH_MAPPINGS = # If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output. # The default value is: YES. -GENERATE_LATEX = YES +GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1,4 +1,4 @@ -Copyright (c) 2014-2020, The Monero Project +Copyright (c) 2014-2021, The Monero Project All rights reserved. @@ -1,6 +1,6 @@ # Monero -Copyright (c) 2014-2020 The Monero Project. +Copyright (c) 2014-2021 The Monero Project. Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Table of Contents diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index a04c63231..5ccfe6fcc 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -28,9 +28,11 @@ #pragma once -#include <limits> -#include <boost/thread.hpp> #include <boost/utility/value_init.hpp> +#include <boost/shared_ptr.hpp> +#include <limits> +#include <functional> +#include <vector> namespace epee { #define STD_TRY_BEGIN() try { @@ -95,16 +97,7 @@ namespace misc_utils return memcmp(&_Left, &_Right, sizeof(_Left)) < 0; } - - inline - bool sleep_no_w(long ms ) - { - boost::this_thread::sleep( - boost::get_system_time() + - boost::posix_time::milliseconds( std::max<long>(ms,0) ) ); - - return true; - } + bool sleep_no_w(long ms ); template <typename T> T get_mid(const T &a, const T &b) diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index cb1388f3b..b03a03cad 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -40,6 +40,7 @@ #include <boost/date_time/posix_time/posix_time.hpp> // TODO #include <boost/thread/condition_variable.hpp> // TODO #include <boost/make_shared.hpp> +#include <boost/thread.hpp> #include "warnings.h" #include "string_tools.h" #include "misc_language.h" @@ -269,8 +270,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) //_dbg3("[sock " << socket().native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); CRITICAL_REGION_LOCAL(self->m_self_refs_lock); //_dbg3("[sock " << socket().native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); - if(m_was_shutdown) - return false; ++m_reference_count; m_self_ref = std::move(self); return true; diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index ddde701ee..6e13c1b73 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -891,12 +891,22 @@ template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::foreach_connection(const callback_t &cb) { CRITICAL_REGION_LOCAL(m_connects_lock); - for(auto& c: m_connects) - { - async_protocol_handler<t_connection_context>* aph = c.second; - if(!cb(aph->get_context_ref())) + std::vector<typename connections_map::mapped_type> conn; + conn.reserve(m_connects.size()); + + auto scope_exit_handler = misc_utils::create_scope_leave_handler([&conn]{ + for (auto &aph: conn) + aph->finish_outer_call(); + }); + + for (auto &e: m_connects) + if (e.second->start_outer_call()) + conn.push_back(e.second); + + for (auto &aph: conn) + if (!cb(aph->get_context_ref())) return false; - } + return true; } //------------------------------------------------------------------------------------------ @@ -907,6 +917,10 @@ bool async_protocol_handler_config<t_connection_context>::for_connection(const b async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); if (!aph) return false; + if (!aph->start_outer_call()) + return false; + auto scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); if(!cb(aph->get_context_ref())) return false; return true; diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index c4cb91130..c615b20e6 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -98,7 +98,12 @@ namespace epee return false; } - return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body)); + static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = { + 65536 * 3, // objects + 65536 * 3, // fields + 65536 * 3, // strings + }; + return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits); } template<class t_request, class t_response, class t_transport> diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 95f0bb410..802e16c1b 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -52,6 +52,11 @@ namespace snprintf(buf, sizeof(buf), "command-%u", command); return on_levin_traffic(context, initiator, sent, error, bytes, buf); } + static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { + 8192, // objects + 16384, // fields + 16384, // strings + }; } namespace epee @@ -77,7 +82,7 @@ namespace epee return false; } serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv)) + if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) { LOG_ERROR("Failed to load_from_binary on command " << command); return false; @@ -124,7 +129,7 @@ namespace epee return false; } typename serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv)) + if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) { on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); @@ -155,7 +160,7 @@ namespace epee return false; } serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff)) + if(!stg_ret.load_from_binary(buff, &default_levin_limits)) { on_levin_traffic(context, true, false, true, buff.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); @@ -205,7 +210,7 @@ namespace epee int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff)) + if(!strg.load_from_binary(in_buff, &default_levin_limits)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in command " << command); @@ -239,7 +244,7 @@ namespace epee int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context) { serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff)) + if(!strg.load_from_binary(in_buff, &default_levin_limits)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in notify " << command); diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index 2aeadf72c..c5d0c48ee 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -24,20 +24,12 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #pragma once -#include <type_traits> - -#include "misc_language.h" #include "portable_storage_base.h" -#include "portable_storage_from_bin.h" -#include "portable_storage_to_json.h" -#include "portable_storage_from_json.h" #include "portable_storage_val_converters.h" +#include "misc_log_ex.h" #include "span.h" -#include "int-util.h" namespace epee { @@ -54,6 +46,13 @@ namespace epee typedef epee::serialization::harray harray; typedef storage_entry meta_entry; + struct limits_t + { + size_t n_objects; + size_t n_fields; + size_t n_strings; // not counting field names + }; + portable_storage(){} virtual ~portable_storage(){} hsection open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist = false); @@ -84,8 +83,8 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target); - bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); } + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); + bool load_from_binary(const std::string& target, const limits_t *limits = NULL); template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -110,83 +109,13 @@ namespace epee }; #pragma pack(pop) }; - inline - bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) - { - TRY_ENTRY(); - std::stringstream ss; - epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines); - buff = ss.str(); - return true; - CATCH_ENTRY("portable_storage::dump_as_json", false) - } - inline - bool portable_storage::load_from_json(const std::string& source) - { - TRY_ENTRY(); - return json::load_from_json(source, *this); - CATCH_ENTRY("portable_storage::load_from_json", false) - } - + template<class trace_policy> bool portable_storage::dump_as_xml(std::string& targetObj, const std::string& root_name) { return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format - } - inline - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source) - { - m_root.m_entries.clear(); - if(source.size() < sizeof(storage_block_header)) - { - LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); - return false; - } - storage_block_header* pbuff = (storage_block_header*)source.data(); - if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || - pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) - ) - { - LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); - return false; - } - if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) - { - LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); - return false; - } - TRY_ENTRY(); - throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); - buf_reader.read(m_root); - return true;//TODO: - CATCH_ENTRY("portable_storage::load_from_binary", false); - } - //--------------------------------------------------------------------------------------------------------------- - inline - hsection portable_storage::open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist) - { - TRY_ENTRY(); - hparent_section = hparent_section ? hparent_section:&m_root; - storage_entry* pentry = find_storage_entry(section_name, hparent_section); - if(!pentry) - { - if(!create_if_notexist) - return nullptr; - return insert_new_section(section_name, hparent_section); - } - CHECK_AND_ASSERT(pentry , nullptr); - //check that section_entry we find is real "CSSection" - if(pentry->type() != typeid(section)) - { - if(create_if_notexist) - *pentry = storage_entry(section());//replace - else - return nullptr; - } - return &boost::get<section>(*pentry); - CATCH_ENTRY("portable_storage::open_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- + } + template<class to_type> struct get_value_visitor: boost::static_visitor<void> { @@ -212,20 +141,6 @@ namespace epee //CATCH_ENTRY("portable_storage::template<>get_value", false); } //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section) - { - //TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(value_name, hparent_section); - if(!pentry) - return false; - - val = *pentry; - return true; - //CATCH_ENTRY("portable_storage::template<>get_value", false); - } - //--------------------------------------------------------------------------------------------------------------- template<class t_value> bool portable_storage::set_value(const std::string& value_name, t_value&& v, hsection hparent_section) { @@ -247,19 +162,6 @@ namespace epee CATCH_ENTRY("portable_storage::template<>set_value", false); } //--------------------------------------------------------------------------------------------------------------- - inline - storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(psection, nullptr); - auto it = psection->m_entries.find(pentry_name); - if(it == psection->m_entries.end()) - return nullptr; - - return &it->second; - CATCH_ENTRY("portable_storage::find_storage_entry", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- template<class entry_type> storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry) { @@ -272,16 +174,6 @@ namespace epee CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr); } //--------------------------------------------------------------------------------------------------------------- - inline - hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection) - { - TRY_ENTRY(); - storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section()); - if(!pse) return nullptr; - return &boost::get<section>(*pse); - CATCH_ENTRY("portable_storage::insert_new_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- template<class to_type> struct get_first_value_visitor: boost::static_visitor<bool> { @@ -335,7 +227,6 @@ namespace epee } }; - template<class t_value> bool portable_storage::get_next_value(harray hval_array, t_value& target) { @@ -393,83 +284,5 @@ namespace epee return true; CATCH_ENTRY("portable_storage::insert_next_value", false); } - //--------------------------------------------------------------------------------------------------------------- - //sections - inline - harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section) - { - TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(sec_name, hparent_section); - if(!pentry) - return nullptr; - if(pentry->type() != typeid(array_entry)) - return nullptr; - array_entry& ar_entry = boost::get<array_entry>(*pentry); - if(ar_entry.type() != typeid(array_entry_t<section>)) - return nullptr; - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); - section* psec = sec_array.get_first_val(); - if(!psec) - return nullptr; - h_child_section = psec; - return &ar_entry; - CATCH_ENTRY("portable_storage::get_first_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(hsec_array, false); - if(hsec_array->type() != typeid(array_entry_t<section>)) - return false; - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); - h_child_section = sec_array.get_next_val(); - if(!h_child_section) - return false; - return true; - CATCH_ENTRY("portable_storage::get_next_section", false); - } - //--------------------------------------------------------------------------------------------------------------- - inline - harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section) - { - TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(sec_name, hparent_section); - if(!pentry) - { - pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>())); - if(!pentry) - return nullptr; - } - if(pentry->type() != typeid(array_entry)) - *pentry = storage_entry(array_entry(array_entry_t<section>())); - - array_entry& ar_entry = boost::get<array_entry>(*pentry); - if(ar_entry.type() != typeid(array_entry_t<section>)) - ar_entry = array_entry(array_entry_t<section>()); - - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); - hinserted_childsection = &sec_array.insert_first_val(section()); - return &ar_entry; - CATCH_ENTRY("portable_storage::insert_first_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(hsec_array, false); - CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), - false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name()); - - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); - hinserted_childsection = &sec_array.insert_next_value(section()); - return true; - CATCH_ENTRY("portable_storage::insert_next_section", false); - } - //--------------------------------------------------------------------------------------------------------------- } } diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index 1676f41fb..ae0be6a34 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -29,10 +29,10 @@ #pragma once #include <boost/variant.hpp> -#include <boost/any.hpp> #include <string> #include <vector> #include <deque> +#include <map> #define PORTABLE_STORAGE_SIGNATUREA 0x01011101 #define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index fa3866d87..9e7b6ec34 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -38,9 +38,6 @@ #else #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL 100 #endif -#define EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL 65536 -#define EPEE_PORTABLE_STORAGE_OBJECT_FIELD_LIMIT_INTERNAL 65536 -#define EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL 65536 // does not include field names namespace epee { @@ -49,21 +46,20 @@ namespace epee template<typename T> struct ps_min_bytes { static constexpr const size_t strict = 4096; // actual low bound - static constexpr const size_t rough = 4096; // when we want to be stricter for DoS prevention }; - template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8, rough = 8; }; - template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8, rough = 8; }; - template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4, rough = 4; }; - template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4, rough = 4; }; - template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2, rough = 2; }; - template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2, rough = 2; }; - template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1, rough = 1; }; - template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1, rough = 1; }; - template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8, rough = 8; }; - template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1, rough = 1; }; - template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2, rough = 16; }; - template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1, rough = 256; }; - template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1, rough = 128; }; + template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; }; + template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; }; + template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; }; struct throwable_buffer_reader { @@ -86,6 +82,7 @@ namespace epee void read(array_entry &ae); template<class t_type> size_t min_bytes() const; + void set_limits(size_t objects, size_t fields, size_t strings); private: struct recursuion_limitation_guard { @@ -109,6 +106,10 @@ namespace epee size_t m_objects; size_t m_fields; size_t m_strings; + + size_t max_objects; + size_t max_fields; + size_t max_strings; }; inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz) @@ -123,6 +124,9 @@ namespace epee m_objects = 0; m_fields = 0; m_strings = 0; + max_objects = std::numeric_limits<size_t>::max(); + max_fields = std::numeric_limits<size_t>::max(); + max_strings = std::numeric_limits<size_t>::max(); } inline void throwable_buffer_reader::read(void* target, size_t count) @@ -173,12 +177,12 @@ namespace epee CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed"); if (std::is_same<type_name, section>()) { - CHECK_AND_ASSERT_THROW_MES(size <= EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL - m_objects, "Too many objects"); + CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects"); m_objects += size; } else if (std::is_same<type_name, std::string>()) { - CHECK_AND_ASSERT_THROW_MES(size <= EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL - m_strings, "Too many strings"); + CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings"); m_strings += size; } @@ -247,7 +251,7 @@ namespace epee inline storage_entry throwable_buffer_reader::read_se<std::string>() { RECURSION_LIMITATION(); - CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL, "Too many strings"); + CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings"); m_strings += 1; return storage_entry(read<std::string>()); } @@ -257,7 +261,7 @@ namespace epee inline storage_entry throwable_buffer_reader::read_se<section>() { RECURSION_LIMITATION(); - CHECK_AND_ASSERT_THROW_MES(m_objects < EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL, "Too many objects"); + CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects"); ++m_objects; section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio storage_entry se(std::move(s)); @@ -310,7 +314,7 @@ namespace epee RECURSION_LIMITATION(); sec.m_entries.clear(); size_t count = read_varint(); - CHECK_AND_ASSERT_THROW_MES(count <= EPEE_PORTABLE_STORAGE_OBJECT_FIELD_LIMIT_INTERNAL - m_fields, "Too many object fields"); + CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields"); m_fields += count; while(count--) { @@ -340,5 +344,12 @@ namespace epee RECURSION_LIMITATION(); CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported"); } + inline + void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings) + { + max_objects = objects; + max_fields = fields; + max_strings = strings; + } } } diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 2b2dc7ff9..3021598f5 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -26,6 +26,7 @@ #pragma once #include <boost/lexical_cast.hpp> +#include <boost/utility/string_ref.hpp> #include <boost/algorithm/string/predicate.hpp> #include "parserse_base_utils.h" #include "file_io_utils.h" diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 39f900c8d..e7250e895 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -32,6 +32,7 @@ #include "parserse_base_utils.h" #include "portable_storage.h" #include "file_io_utils.h" +#include "span.h" namespace epee { @@ -85,10 +86,10 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff) + bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL) { portable_storage ps; - bool rs = ps.load_from_binary(binary_buff); + bool rs = ps.load_from_binary(binary_buff, limits); if(!rs) return false; diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 49a7be185..b82cf532b 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -32,6 +32,7 @@ #include "misc_language.h" #include "portable_storage_base.h" #include "portable_storage_bin_utils.h" +#include "misc_log_ex.h" namespace epee { diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index e54cda828..96b0c024c 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -28,12 +28,17 @@ #pragma once -#include <time.h> #include <boost/regex.hpp> #include "misc_language.h" #include "portable_storage_base.h" +#include "parserse_base_utils.h" #include "warnings.h" +#include "misc_log_ex.h" + +#include <boost/lexical_cast.hpp> +#include <typeinfo> +#include <iomanip> namespace epee { diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 5e101a86a..641d4718e 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -29,7 +29,9 @@ add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp - int-util.cpp portable_storage.cpp) + int-util.cpp portable_storage.cpp + misc_language.cpp + ) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) add_library(epee_readline STATIC readline_buffer.cpp) @@ -71,3 +73,6 @@ if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) PRIVATE ${GNU_READLINE_LIBRARY}) endif() + +target_include_directories(epee PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include") + diff --git a/contrib/epee/src/misc_language.cpp b/contrib/epee/src/misc_language.cpp new file mode 100644 index 000000000..6e8f2daef --- /dev/null +++ b/contrib/epee/src/misc_language.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_language.h" + +#include <boost/thread.hpp> + +namespace epee +{ +namespace misc_utils +{ + bool sleep_no_w(long ms ) + { + boost::this_thread::sleep( + boost::get_system_time() + + boost::posix_time::milliseconds( std::max<long>(ms,0) ) ); + + return true; + } +} +} diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index 4534deff3..c3c9ccc02 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -1,10 +1,43 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #include "byte_slice.h" #include "byte_stream.h" #include "misc_log_ex.h" #include "span.h" #include "storages/portable_storage.h" +#include "storages/portable_storage_to_json.h" +#include "storages/portable_storage_from_json.h" #include "storages/portable_storage_to_bin.h" +#include "storages/portable_storage_from_bin.h" + +#include <boost/utility/string_ref.hpp> + +#include <string> +#include <sstream> namespace epee { @@ -25,5 +58,187 @@ namespace serialization return true; CATCH_ENTRY("portable_storage::store_to_binary", false) } + + bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) + { + TRY_ENTRY(); + std::stringstream ss; + epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines); + buff = ss.str(); + return true; + CATCH_ENTRY("portable_storage::dump_as_json", false) + } + + bool portable_storage::load_from_json(const std::string& source) + { + TRY_ENTRY(); + return json::load_from_json(source, *this); + CATCH_ENTRY("portable_storage::load_from_json", false) + } + + bool portable_storage::load_from_binary(const std::string& target, const limits_t *limits) + { + return load_from_binary(epee::strspan<uint8_t>(target), limits); + } + + bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) + { + m_root.m_entries.clear(); + if(source.size() < sizeof(storage_block_header)) + { + LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); + return false; + } + storage_block_header* pbuff = (storage_block_header*)source.data(); + if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || + pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) + ) + { + LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); + return false; + } + if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) + { + LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); + return false; + } + TRY_ENTRY(); + throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); + if (limits) + buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); + buf_reader.read(m_root); + return true;//TODO: + CATCH_ENTRY("portable_storage::load_from_binary", false); + } + + hsection portable_storage::open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist) + { + TRY_ENTRY(); + hparent_section = hparent_section ? hparent_section:&m_root; + storage_entry* pentry = find_storage_entry(section_name, hparent_section); + if(!pentry) + { + if(!create_if_notexist) + return nullptr; + return insert_new_section(section_name, hparent_section); + } + CHECK_AND_ASSERT(pentry , nullptr); + //check that section_entry we find is real "CSSection" + if(pentry->type() != typeid(section)) + { + if(create_if_notexist) + *pentry = storage_entry(section());//replace + else + return nullptr; + } + return &boost::get<section>(*pentry); + CATCH_ENTRY("portable_storage::open_section", nullptr); + } + + bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section) + { + //TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(value_name, hparent_section); + if(!pentry) + return false; + + val = *pentry; + return true; + //CATCH_ENTRY("portable_storage::template<>get_value", false); + } + + storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(psection, nullptr); + auto it = psection->m_entries.find(pentry_name); + if(it == psection->m_entries.end()) + return nullptr; + + return &it->second; + CATCH_ENTRY("portable_storage::find_storage_entry", nullptr); + } + + hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection) + { + TRY_ENTRY(); + storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section()); + if(!pse) return nullptr; + return &boost::get<section>(*pse); + CATCH_ENTRY("portable_storage::insert_new_section", nullptr); + } + + harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section) + { + TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(sec_name, hparent_section); + if(!pentry) + return nullptr; + if(pentry->type() != typeid(array_entry)) + return nullptr; + array_entry& ar_entry = boost::get<array_entry>(*pentry); + if(ar_entry.type() != typeid(array_entry_t<section>)) + return nullptr; + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); + section* psec = sec_array.get_first_val(); + if(!psec) + return nullptr; + h_child_section = psec; + return &ar_entry; + CATCH_ENTRY("portable_storage::get_first_section", nullptr); + } + + bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(hsec_array, false); + if(hsec_array->type() != typeid(array_entry_t<section>)) + return false; + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); + h_child_section = sec_array.get_next_val(); + if(!h_child_section) + return false; + return true; + CATCH_ENTRY("portable_storage::get_next_section", false); + } + + harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section) + { + TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(sec_name, hparent_section); + if(!pentry) + { + pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>())); + if(!pentry) + return nullptr; + } + if(pentry->type() != typeid(array_entry)) + *pentry = storage_entry(array_entry(array_entry_t<section>())); + + array_entry& ar_entry = boost::get<array_entry>(*pentry); + if(ar_entry.type() != typeid(array_entry_t<section>)) + ar_entry = array_entry(array_entry_t<section>()); + + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); + hinserted_childsection = &sec_array.insert_first_val(section()); + return &ar_entry; + CATCH_ENTRY("portable_storage::insert_first_section", nullptr); + } + + bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(hsec_array, false); + CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), + false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name()); + + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); + hinserted_childsection = &sec_array.insert_next_value(section()); + return true; + CATCH_ENTRY("portable_storage::insert_next_section", false); + } } } diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index bcf499963..1047d1696 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -6,6 +6,7 @@ #include <boost/thread/lock_guard.hpp> #include <boost/algorithm/string.hpp> +static bool same_as_last_line(const std::string&); static void install_line_handler(); static void remove_line_handler(); @@ -175,8 +176,11 @@ static void handle_line(char* line) boost::trim_right(test_line); if(!test_line.empty()) { - add_history(test_line.c_str()); - history_set_pos(history_length); + if (!same_as_last_line(test_line)) + { + add_history(test_line.c_str()); + history_set_pos(history_length); + } if (test_line == "exit" || test_line == "q") exit = true; } @@ -192,6 +196,16 @@ static void handle_line(char* line) return; } +// same_as_last_line returns true, if the last line in the history is +// equal to test_line. +static bool same_as_last_line(const std::string& test_line) +{ + // Note that state->offset == state->length, when a new line was entered. + HISTORY_STATE* state = history_get_history_state(); + return state->length > 0 + && test_line.compare(state->entries[state->length-1]->line) == 0; +} + static char* completion_matches(const char* text, int state) { static size_t list_index; diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index efd43c231..5c88c3727 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -14,8 +14,8 @@ then exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/docs/ANONYMITY_NETWORKS.md b/docs/ANONYMITY_NETWORKS.md index 3337b5fc3..f8d08b05f 100644 --- a/docs/ANONYMITY_NETWORKS.md +++ b/docs/ANONYMITY_NETWORKS.md @@ -36,10 +36,6 @@ with additional exclusive IPv4 address(es). ## Usage -Anonymity networks have no seed nodes (the feature is still considered -experimental), so a user must specify an address. If configured properly, -additional peers can be found through typical p2p peerlist sharing. - ### Outbound Connections Connecting to an anonymous address requires the command line option @@ -54,8 +50,9 @@ separate process. On most systems the configuration will look like: which tells `monerod` that ".onion" p2p addresses can be forwarded to a socks proxy at IP 127.0.0.1 port 9050 with a max of 10 outgoing connections and ".b32.i2p" p2p addresses can be forwarded to a socks proxy at IP 127.0.0.1 port -9000 with the default max outgoing connections. Since there are no seed nodes -for anonymity connections, peers must be manually specified: +9000 with the default max outgoing connections. + +If desired, peers can be manually specified: ``` --add-exclusive-node rveahdfho7wo4b2m.onion:28083 diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index a8916a7d0..7ae4ba750 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -38,8 +38,10 @@ find_package(Miniupnpc REQUIRED) message(STATUS "Using in-tree miniupnpc") +set(UPNPC_NO_INSTALL TRUE CACHE BOOL "Disable miniupnp installation" FORCE) add_subdirectory(miniupnp/miniupnpc) set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external") +set_property(TARGET libminiupnpc-static PROPERTY POSITION_INDEPENDENT_CODE ON) if(MSVC) set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267") elseif(NOT MSVC) diff --git a/external/miniupnp b/external/miniupnp -Subproject 4c700e09526a7d546394e85628c57e9490feefa +Subproject 544e6fcc73c5ad9af48a8985c94f0f1d742ef2e diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 4f4efcd81..c31d1dd96 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -484,36 +484,12 @@ std::string get_account_address_as_str_from_url(const std::string& url, bool& dn return dns_confirm(url, addresses, dnssec_valid); } -namespace -{ - bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b) - { - if (a.size() != b.size()) return false; - - for (const auto& record_in_a : a) - { - bool ok = false; - for (const auto& record_in_b : b) - { - if (record_in_a == record_in_b) - { - ok = true; - break; - } - } - if (!ok) return false; - } - - return true; - } -} - bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std::vector<std::string> &dns_urls) { // Prevent infinite recursion when distributing if (dns_urls.empty()) return false; - std::vector<std::vector<std::string> > records; + std::vector<std::set<std::string> > records; records.resize(dns_urls.size()); size_t first_index = crypto::rand_idx(dns_urls.size()); @@ -525,7 +501,9 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std for (size_t n = 0; n < dns_urls.size(); ++n) { tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ - records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); + const auto res = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); + for (const auto &s: res) + records[n].insert(s); }); } waiter.wait(); @@ -568,29 +546,31 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std return false; } - int good_records_index = -1; - for (size_t i = 0; i < records.size() - 1; ++i) + typedef std::map<std::set<std::string>, uint32_t> map_t; + map_t record_count; + for (const auto &e: records) { - if (records[i].size() == 0) continue; + if (!e.empty()) + ++record_count[e]; + } - for (size_t j = i + 1; j < records.size(); ++j) - { - if (dns_records_match(records[i], records[j])) - { - good_records_index = i; - break; - } - } - if (good_records_index >= 0) break; + map_t::const_iterator good_record = record_count.end(); + for (map_t::const_iterator i = record_count.begin(); i != record_count.end(); ++i) + { + if (good_record == record_count.end() || i->second > good_record->second) + good_record = i; } - if (good_records_index < 0) + MDEBUG("Found " << (good_record == record_count.end() ? 0 : good_record->second) << "/" << dns_urls.size() << " matching records from " << num_valid_records << " valid records"); + if (good_record == record_count.end() || good_record->second < dns_urls.size() / 2 + 1) { - LOG_PRINT_L0("WARNING: no two DNS TXT records matched"); + LOG_PRINT_L0("WARNING: no majority of DNS TXT records matched (only " << good_record->second << "/" << dns_urls.size() << ")"); return false; } - good_records = records[good_records_index]; + good_records = {}; + for (const auto &s: good_record->first) + good_records.push_back(s); return true; } diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 61a76f5da..af38d7a54 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -48,7 +48,10 @@ namespace tools static const std::vector<std::string> dns_urls = { "updates.moneropulse.org", "updates.moneropulse.net", - "updates.moneropulse.co", + "updates.moneropulse.fr", + "updates.moneropulse.de", + "updates.moneropulse.no", + "updates.moneropulse.ch", "updates.moneropulse.se" }; diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 2cb28b2b1..6327f7379 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -194,7 +194,6 @@ #define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase #define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved #define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved -//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED #define RPC_CREDITS_PER_HASH_SCALE ((float)(1<<24)) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index cfd4a1596..71d09d08c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -338,10 +338,6 @@ namespace cryptonote } context.m_remote_blockchain_height = hshd.current_height; context.m_pruning_seed = hshd.pruning_seed; -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as<epee::net_utils::ipv4_network_address>().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); - LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); -#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -549,6 +545,7 @@ namespace cryptonote } std::vector<tx_blob_entry> have_tx; + have_tx.reserve(new_block.tx_hashes.size()); // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -557,6 +554,7 @@ namespace cryptonote // Also, remember to pepper some whitespace changes around to bother // moneromooo ... only because I <3 him. std::vector<uint64_t> need_tx_indices; + need_tx_indices.reserve(new_block.tx_hashes.size()); transaction tx; crypto::hash tx_hash; @@ -829,6 +827,7 @@ namespace cryptonote } std::vector<crypto::hash> txids; + txids.reserve(b.tx_hashes.size()); NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response; fluffy_response.b.block = t_serializable_object_to_blob(b); fluffy_response.current_blockchain_height = arg.current_blockchain_height; @@ -2189,6 +2188,7 @@ skip: if (span.second > 0) { is_next = true; + req.blocks.reserve(hashes.size()); for (const auto &hash: hashes) { req.blocks.push_back(hash); @@ -2247,6 +2247,7 @@ skip: if (span.second > 0) { is_next = true; + req.blocks.reserve(hashes.size()); for (const auto &hash: hashes) { req.blocks.push_back(hash); @@ -2280,6 +2281,7 @@ skip: return false; } + req.blocks.reserve(req.blocks.size() + span.second); for (size_t n = 0; n < span.second; ++n) { req.blocks.push_back(context.m_needed_objects[n].first); @@ -2579,6 +2581,7 @@ skip: } context.m_needed_objects.clear(); + context.m_needed_objects.reserve(arg.m_block_ids.size()); uint64_t added = 0; std::unordered_set<crypto::hash> blocks_found; bool first = true; diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index c23a4d2fe..1e9f3e399 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -51,14 +51,6 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.p2p.tx" -namespace -{ - int get_command_from_message(const epee::byte_slice &msg) - { - return msg.size() >= sizeof(epee::levin::bucket_head2) ? SWAP32LE(((epee::levin::bucket_head2*)msg.data())->m_command) : 0; - } -} - namespace cryptonote { namespace levin @@ -212,7 +204,7 @@ namespace levin { const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); p2p.for_connection(destination, [&blob](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob)); + on_levin_traffic(context, true, true, false, blob.size(), NOTIFY_NEW_TRANSACTIONS::ID); return true; }); return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 04feb55fd..5a7d4dd4e 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -2275,6 +2275,7 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; + tools::success_msg_writer() << "Remote Host Peer_ID State Prune_Seed Height DL kB/s, Queued Blocks / MB"; for (const auto &p: res.peers) { std::string address = epee::string_tools::pad_string(p.info.address, 24); diff --git a/src/debug_utilities/dns_checks.cpp b/src/debug_utilities/dns_checks.cpp index 76b66c6cb..138cd4fc1 100644 --- a/src/debug_utilities/dns_checks.cpp +++ b/src/debug_utilities/dns_checks.cpp @@ -131,7 +131,7 @@ int main(int argc, char* argv[]) lookup(LOOKUP_A, {"seeds.moneroseeds.se", "seeds.moneroseeds.ae.org", "seeds.moneroseeds.ch", "seeds.moneroseeds.li"}); - lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se"}); + lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se", "updates.moneropulse.fr", "updates.moneropulse.de", "updates.moneropulse.no", "updates.moneropulse.ch"}); lookup(LOOKUP_TXT, {"checkpoints.moneropulse.org", "checkpoints.moneropulse.net", "checkpoints.moneropulse.co", "checkpoints.moneropulse.se"}); diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index bf8793aa2..1aa869e45 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -129,7 +129,7 @@ namespace Language if ((*it).size() < unique_prefix_length)
{
if (flags & ALLOW_SHORT_WORDS)
- MWARNING(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
+ MINFO(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
else
throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
}
diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 8dd551d1e..84cc1581e 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -166,6 +166,7 @@ namespace nodetool const command_line::arg_descriptor<bool> arg_pad_transactions = { "pad-transactions", "Pad relayed transactions to help defend against traffic volume analysis", false }; + const command_line::arg_descriptor<uint32_t> arg_max_connections_per_ip = {"max-connections-per-ip", "Maximum number of connections allowed from the same IP address", 1}; boost::optional<std::vector<proxy>> get_proxies(boost::program_options::variables_map const& vm) { diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 59a6e5091..db931122e 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -258,7 +258,8 @@ namespace nodetool m_igd(no_igd), m_offline(false), is_closing(false), - m_network_id() + m_network_id(), + max_connections(1) {} virtual ~node_server(); @@ -517,6 +518,8 @@ namespace nodetool epee::net_utils::ssl_support_t m_ssl_support; bool m_enable_dns_blocklist; + + uint32_t max_connections; }; const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s @@ -551,6 +554,7 @@ namespace nodetool extern const command_line::arg_descriptor<int64_t> arg_limit_rate_down; extern const command_line::arg_descriptor<int64_t> arg_limit_rate; extern const command_line::arg_descriptor<bool> arg_pad_transactions; + extern const command_line::arg_descriptor<uint32_t> arg_max_connections_per_ip; } POP_WARNINGS diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index b8bd7b2a7..07b45b9bd 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -131,6 +131,7 @@ namespace nodetool command_line::add_arg(desc, arg_limit_rate_down); command_line::add_arg(desc, arg_limit_rate); command_line::add_arg(desc, arg_pad_transactions); + command_line::add_arg(desc, arg_max_connections_per_ip); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -234,6 +235,7 @@ namespace nodetool return false; const time_t now = time(nullptr); + bool added = false; CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); time_t limit; @@ -244,7 +246,10 @@ namespace nodetool const std::string host_str = addr.host_str(); auto it = m_blocked_hosts.find(host_str); if (it == m_blocked_hosts.end()) + { m_blocked_hosts[host_str] = limit; + added = true; + } else if (it->second < limit || !add_only) it->second = limit; @@ -275,7 +280,10 @@ namespace nodetool conns.clear(); } - MCLOG_CYAN(el::Level::Info, "global", "Host " << host_str << " blocked."); + if (added) + MCLOG_CYAN(el::Level::Info, "global", "Host " << host_str << " blocked."); + else + MINFO("Host " << host_str << " block time updated."); return true; } //----------------------------------------------------------------------------------- @@ -608,6 +616,8 @@ namespace nodetool return false; } + max_connections = command_line::get_arg(vm, arg_max_connections_per_ip); + return true; } //----------------------------------------------------------------------------------- @@ -887,32 +897,6 @@ namespace nodetool for(const auto& p: m_command_line_peers) m_network_zones.at(p.adr.get_zone()).m_peerlist.append_with_peer_white(p); -// all peers are now setup -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - for (auto& zone : m_network_zones) - { - std::list<peerlist_entry> plw; - while (zone.second.m_peerlist.get_white_peers_count()) - { - plw.push_back(peerlist_entry()); - zone.second.m_peerlist.get_white_peer_by_index(plw.back(), 0); - zone.second.m_peerlist.remove_from_peer_white(plw.back()); - } - for (auto &e:plw) - zone.second.m_peerlist.append_with_peer_white(e); - - std::list<peerlist_entry> plg; - while (zone.second.m_peerlist.get_gray_peers_count()) - { - plg.push_back(peerlist_entry()); - zone.second.m_peerlist.get_gray_peer_by_index(plg.back(), 0); - zone.second.m_peerlist.remove_from_peer_gray(plg.back()); - } - for (auto &e:plg) - zone.second.m_peerlist.append_with_peer_gray(e); - } -#endif - //only in case if we really sure that we have external visible ip m_have_address = true; @@ -1155,6 +1139,7 @@ namespace nodetool pi = context.peer_id = rsp.node_data.peer_id; context.m_rpc_port = rsp.node_data.rpc_port; context.m_rpc_credits_per_hash = rsp.node_data.rpc_credits_per_hash; + context.support_flags = rsp.node_data.support_flags; const auto azone = context.m_remote_address.get_zone(); network_zone& zone = m_network_zones.at(azone); zone.m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash); @@ -1188,10 +1173,11 @@ namespace nodetool } else if (!just_take_peerlist) { - try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) - { - flags_context.support_flags = support_flags; - }); + if (context_.support_flags == 0) + try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) + { + flags_context.support_flags = support_flags; + }); } return hsh_result; @@ -2015,6 +2001,8 @@ namespace nodetool boost::split(ips, record, boost::is_any_of(";")); for (const auto &ip: ips) { + if (ip.empty()) + continue; const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(ip, 0); if (!parsed_addr) { @@ -2118,10 +2106,6 @@ namespace nodetool continue; } local_peerlist[i].last_seen = 0; - -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); -#endif } return true; } @@ -2167,6 +2151,7 @@ namespace nodetool node_data.rpc_port = zone.m_can_pingback ? m_rpc_port : 0; node_data.rpc_credits_per_hash = zone.m_can_pingback ? m_rpc_credits_per_hash : 0; node_data.network_id = m_network_id; + node_data.support_flags = zone.m_config.m_support_flags; return true; } //----------------------------------------------------------------------------------- @@ -2537,6 +2522,7 @@ namespace nodetool context.m_in_timedsync = false; context.m_rpc_port = arg.node_data.rpc_port; context.m_rpc_credits_per_hash = arg.node_data.rpc_credits_per_hash; + context.support_flags = arg.node_data.support_flags; if(arg.node_data.my_port && zone.m_can_pingback) { @@ -2570,10 +2556,11 @@ namespace nodetool }); } - try_get_support_flags(context, [](p2p_connection_context& flags_context, const uint32_t& support_flags) - { - flags_context.support_flags = support_flags; - }); + if (context.support_flags == 0) + try_get_support_flags(context, [](p2p_connection_context& flags_context, const uint32_t& support_flags) + { + flags_context.support_flags = support_flags; + }); //fill response zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); @@ -2839,8 +2826,7 @@ namespace nodetool if (address.get_zone() != epee::net_utils::zone::public_) return false; // Unable to determine how many connections from host - const size_t max_connections = 1; - size_t count = 0; + uint32_t count = 0; m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 37a85d526..1cf4cb026 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -38,10 +38,6 @@ #include "net/i2p_address.h" #include "p2p/p2p_protocol_defs.h" -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED -#include "common/pruning.h" -#endif - BOOST_CLASS_VERSION(nodetool::peerlist_entry, 3) namespace boost @@ -228,12 +224,6 @@ namespace boost return; } a & pl.pruning_seed; -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - if (!typename Archive::is_saving()) - { - pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); - } -#endif if (ver < 2) { if (!typename Archive::is_saving()) diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index b439dc47e..12763d4ee 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -188,6 +188,7 @@ namespace nodetool uint16_t rpc_port; uint32_t rpc_credits_per_hash; peerid_type peer_id; + uint32_t support_flags; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_VAL_POD_AS_BLOB(network_id) @@ -195,6 +196,7 @@ namespace nodetool KV_SERIALIZE(my_port) KV_SERIALIZE_OPT(rpc_port, (uint16_t)(0)) KV_SERIALIZE_OPT(rpc_credits_per_hash, (uint32_t)0) + KV_SERIALIZE_OPT(support_flags, (uint32_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 5b2043de6..db228dd94 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -452,7 +452,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); - res.target_height = m_core.get_target_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase @@ -2913,7 +2913,7 @@ namespace cryptonote crypto::hash top_hash; m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height - res.target_height = m_core.get_target_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index dad4aab6c..c3df8a66f 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -195,7 +195,7 @@ namespace const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); - const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); + const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw] [<filename>]"); const char* USAGE_SET_LOG("set_log <level>|{+,-,}<categories>"); const char* USAGE_ACCOUNT("account\n" " account new <label text with white spaces allowed>\n" @@ -3341,7 +3341,8 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign_transfer, _1), tr(USAGE_SIGN_TRANSFER), - tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); + tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.\n" + "Use the parameter <filename> to specify the file to read from. If not specified, the default \"unsigned_monero_tx\" will be used.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::submit_transfer, _1), tr("Submit a signed transaction from a file.")); @@ -4563,7 +4564,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) password = *r; welcome = true; // if no block_height is specified, assume its a new account and start it "now" - if(m_wallet->get_refresh_from_block_height() == 0) { + if (command_line::is_arg_defaulted(vm, arg_restore_height)) { { tools::scoped_message_writer wrt = tools::msg_writer(); wrt << tr("No restore height is specified.") << " "; @@ -4809,9 +4810,14 @@ bool simple_wallet::try_connect_to_daemon(bool silent, uint32_t* version) if (!m_wallet->check_connection(version)) { if (!silent) - fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " << - tr("Daemon either is not started or wrong port was passed. " - "Please make sure daemon is running or change the daemon address using the 'set_daemon' command."); + { + if (m_wallet->is_offline()) + fail_msg_writer() << tr("wallet failed to connect to daemon, because it is set to offline mode"); + else + fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " << + tr("Daemon either is not started or wrong port was passed. " + "Please make sure daemon is running or change the daemon address using the 'set_daemon' command."); + } return false; } if (!m_allow_mismatched_daemon_version && ((*version >> 16) != CORE_RPC_VERSION_MAJOR)) @@ -7930,19 +7936,33 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } - if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) + + bool export_raw = false; + std::string unsigned_filename = "unsigned_monero_tx"; + if (args_.size() > 2 || (args_.size() == 2 && args_[0] != "export_raw")) { PRINT_USAGE(USAGE_SIGN_TRANSFER); return true; } + else if (args_.size() == 2) + { + export_raw = true; + unsigned_filename = args_[1]; + } + else if (args_.size() == 1) + { + if (args_[0] == "export_raw") + export_raw = true; + else + unsigned_filename = args_[0]; + } SCOPED_WALLET_UNLOCK(); - const bool export_raw = args_.size() == 1; std::vector<tools::wallet2::pending_tx> ptx; try { - bool r = m_wallet->sign_tx("unsigned_monero_tx", "signed_monero_tx", ptx, [&](const tools::wallet2::unsigned_tx_set &tx){ return accept_loaded_tx(tx); }, export_raw); + bool r = m_wallet->sign_tx(unsigned_filename, "signed_monero_tx", ptx, [&](const tools::wallet2::unsigned_tx_set &tx){ return accept_loaded_tx(tx); }, export_raw); if (!r) { fail_msg_writer() << tr("Failed to sign transaction"); @@ -9342,7 +9362,7 @@ bool simple_wallet::run() refresh_main(0, ResetNone, true); - m_auto_refresh_enabled = m_wallet->auto_refresh(); + m_auto_refresh_enabled = !m_wallet->is_offline() && m_wallet->auto_refresh(); m_idle_thread = boost::thread([&]{wallet_idle_thread();}); message_writer(console_color_green, false) << "Background refresh thread started"; diff --git a/src/wallet/api/transaction_history.cpp b/src/wallet/api/transaction_history.cpp index b6d52c58d..d8e4aab65 100644 --- a/src/wallet/api/transaction_history.cpp +++ b/src/wallet/api/transaction_history.cpp @@ -139,6 +139,7 @@ void TransactionHistoryImpl::refresh() ti->m_paymentid = payment_id; ti->m_coinbase = pd.m_coinbase; ti->m_amount = pd.m_amount; + ti->m_fee = pd.m_fee; ti->m_direction = TransactionInfo::Direction_In; ti->m_hash = string_tools::pod_to_hex(pd.m_tx_hash); ti->m_blockheight = pd.m_block_height; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 9acc8484c..4f923ce54 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -1168,7 +1168,7 @@ bool WalletImpl::submitTransaction(const string &fileName) { return true; } -bool WalletImpl::exportKeyImages(const string &filename) +bool WalletImpl::exportKeyImages(const string &filename, bool all) { if (m_wallet->watch_only()) { @@ -1178,7 +1178,7 @@ bool WalletImpl::exportKeyImages(const string &filename) try { - if (!m_wallet->export_key_images(filename)) + if (!m_wallet->export_key_images(filename), all) { setStatusError(tr("failed to save file ") + filename); return false; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 3bf3e759b..e501d3943 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -164,7 +164,7 @@ public: virtual PendingTransaction * createSweepUnmixableTransaction() override; bool submitTransaction(const std::string &fileName) override; virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename) override; - bool exportKeyImages(const std::string &filename) override; + bool exportKeyImages(const std::string &filename, bool all = false) override; bool importKeyImages(const std::string &filename) override; virtual void disposeTransaction(PendingTransaction * t) override; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 44928a422..b40b6763f 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -901,9 +901,10 @@ struct Wallet /*! * \brief exportKeyImages - exports key images to file * \param filename + * \param all - export all key images or only those that have not yet been exported * \return - true on success */ - virtual bool exportKeyImages(const std::string &filename) = 0; + virtual bool exportKeyImages(const std::string &filename, bool all = false) = 0; /*! * \brief importKeyImages - imports key images from file diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 48a602bf3..a576c267c 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -32,6 +32,8 @@ #include "rpc/rpc_payment_costs.h" #include "storages/http_abstract_invoke.h" +#include <boost/thread.hpp> + #define RETURN_ON_RPC_RESPONSE_ERROR(r, error, res, method) \ do { \ CHECK_AND_ASSERT_MES(error.code == 0, error.message, error.message); \ diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 51b7f01dd..f5e3fca5f 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -30,6 +30,7 @@ #include <string> #include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> #include "include_base_utils.h" #include "net/abstract_http_client.h" #include "rpc/core_rpc_server_commands_defs.h" diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index d2bcc04d6..157059c3d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -6084,6 +6084,14 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo found->second += utx.second.m_change; } } + + for (const auto& utx: m_unconfirmed_payments) + { + if (utx.second.m_pd.m_subaddr_index.major == index_major) + { + amount_per_subaddr[utx.second.m_pd.m_subaddr_index.minor] += utx.second.m_pd.m_amount; + } + } } return amount_per_subaddr; } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 1488c6976..e5a5136a4 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1551,6 +1551,7 @@ private: void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); void enable_dns(bool enable) { m_use_dns = enable; } void set_offline(bool offline = true); + bool is_offline() const { return m_offline; } uint64_t credits() const { return m_rpc_payment_state.credits; } void credit_report(uint64_t &expected_spent, uint64_t &discrepancy) const { expected_spent = m_rpc_payment_state.expected_spent; discrepancy = m_rpc_payment_state.discrepancy; } diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 03db8b70f..13de73f8c 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -4438,7 +4438,14 @@ public: wal->stop(); }); - wal->refresh(wal->is_trusted_daemon()); + try + { + wal->refresh(wal->is_trusted_daemon()); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Initial refresh failed: ") << e.what()); + } // if we ^C during potentially length load/refresh, there's no server loop yet if (quit) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 224784a18..85fa8f1dd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -80,17 +80,22 @@ file(COPY data/signed_monero_tx DESTINATION data) -add_subdirectory(core_tests) -add_subdirectory(fuzz) -add_subdirectory(crypto) -add_subdirectory(functional_tests) -add_subdirectory(performance_tests) -add_subdirectory(core_proxy) -add_subdirectory(unit_tests) -add_subdirectory(difficulty) -add_subdirectory(block_weight) -add_subdirectory(hash) -add_subdirectory(net_load_tests) +if (CMAKE_BUILD_TYPE STREQUAL "fuzz" OR OSSFUZZ) + add_subdirectory(fuzz) +else () + add_subdirectory(core_tests) + add_subdirectory(fuzz) + add_subdirectory(crypto) + add_subdirectory(functional_tests) + add_subdirectory(performance_tests) + add_subdirectory(core_proxy) + add_subdirectory(unit_tests) + add_subdirectory(difficulty) + add_subdirectory(block_weight) + add_subdirectory(hash) + add_subdirectory(net_load_tests) +endif() + if (BUILD_GUI_DEPS) add_subdirectory(libwallet_api_tests) endif() diff --git a/tests/data/fuzz/utf8/UTF8_1 b/tests/data/fuzz/utf8/UTF8_1 new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/data/fuzz/utf8/UTF8_1 diff --git a/tests/data/fuzz/utf8/UTF8_2 b/tests/data/fuzz/utf8/UTF8_2 Binary files differnew file mode 100644 index 000000000..bb6982c4f --- /dev/null +++ b/tests/data/fuzz/utf8/UTF8_2 diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index a599f86f8..0cf1740ad 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -218,3 +218,13 @@ set_property(TARGET tx-extra_fuzz_tests PROPERTY FOLDER "tests") +monero_add_minimal_executable(utf8_fuzz_tests utf8.cpp fuzzer.cpp) +target_link_libraries(utf8_fuzz_tests + PRIVATE + common + epee + ${Boost_THREAD_LIBRARY} + ${Boost_CHRONO_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES} + $ENV{LIB_FUZZING_ENGINE}) diff --git a/tests/fuzz/utf8.cpp b/tests/fuzz/utf8.cpp new file mode 100644 index 000000000..bf304a351 --- /dev/null +++ b/tests/fuzz/utf8.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2017-2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "common/utf8.h" +#include "fuzzer.h" + +BEGIN_INIT_SIMPLE_FUZZER() +END_INIT_SIMPLE_FUZZER() + +BEGIN_SIMPLE_FUZZER() + tools::utf8canonical(std::string((const char*)buf, len), [](wint_t c)->wint_t { return c; }); +END_SIMPLE_FUZZER() diff --git a/tests/unit_tests/epee_boosted_tcp_server.cpp b/tests/unit_tests/epee_boosted_tcp_server.cpp index 71098f612..e99666eb1 100644 --- a/tests/unit_tests/epee_boosted_tcp_server.cpp +++ b/tests/unit_tests/epee_boosted_tcp_server.cpp @@ -37,6 +37,7 @@ #include "include_base_utils.h" #include "string_tools.h" #include "net/abstract_tcp_server2.h" +#include "net/levin_protocol_handler_async.h" namespace { @@ -132,3 +133,152 @@ TEST(boosted_tcp_server, worker_threads_are_exception_resistant) ASSERT_TRUE(srv.timed_wait_server_stop(5 * 1000)); ASSERT_TRUE(srv.deinit_server()); } + + +TEST(test_epee_connection, test_lifetime) +{ + struct context_t: epee::net_utils::connection_context_base { + static constexpr size_t get_max_bytes(int) noexcept { return -1; } + static constexpr int handshake_command() noexcept { return 1001; } + static constexpr bool handshake_complete() noexcept { return true; } + }; + + struct command_handler_t: epee::levin::levin_commands_handler<context_t> { + size_t delay; + command_handler_t(size_t delay = 0): delay(delay) {} + virtual int invoke(int, const epee::span<const uint8_t>, epee::byte_slice&, context_t&) override { epee::misc_utils::sleep_no_w(delay); return {}; } + virtual int notify(int, const epee::span<const uint8_t>, context_t&) override { return {}; } + virtual void callback(context_t&) override {} + virtual void on_connection_new(context_t&) override {} + virtual void on_connection_close(context_t&) override {} + virtual ~command_handler_t() override {} + static void destroy(epee::levin::levin_commands_handler<context_t>* ptr) { delete ptr; } + }; + + using handler_t = epee::levin::async_protocol_handler<context_t>; + using connection_t = epee::net_utils::connection<handler_t>; + using connection_ptr = boost::shared_ptr<connection_t>; + using shared_state_t = typename connection_t::shared_state; + using shared_state_ptr = std::shared_ptr<shared_state_t>; + using tag_t = boost::uuids::uuid; + using tags_t = std::vector<tag_t>; + using io_context_t = boost::asio::io_service; + using endpoint_t = boost::asio::ip::tcp::endpoint; + using work_t = boost::asio::io_service::work; + using work_ptr = std::shared_ptr<work_t>; + using workers_t = std::vector<std::thread>; + using server_t = epee::net_utils::boosted_tcp_server<handler_t>; + + io_context_t io_context; + work_ptr work(std::make_shared<work_t>(io_context)); + + workers_t workers; + while (workers.size() < 4) { + workers.emplace_back([&io_context]{ + io_context.run(); + }); + } + + endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 5262); + server_t server(epee::net_utils::e_connection_type_P2P); + server.init_server(endpoint.port(), + endpoint.address().to_string(), + 0, + "", + false, + true, + epee::net_utils::ssl_support_t::e_ssl_support_disabled + ); + server.run_server(2, false); + server.get_config_shared()->set_handler(new command_handler_t, &command_handler_t::destroy); + + io_context.post([&io_context, &work, &endpoint, &server]{ + auto scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&work]{ + work.reset(); + }); + + shared_state_ptr shared_state(std::make_shared<shared_state_t>()); + shared_state->set_handler(new command_handler_t, &command_handler_t::destroy); + + auto create_connection = [&io_context, &endpoint, &shared_state] { + connection_ptr conn(new connection_t(io_context, shared_state, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + context_t context; + conn->get_context(context); + auto tag = context.m_connection_id; + return tag; + }; + + ASSERT_TRUE(shared_state->get_connections_count() == 0); + auto tag = create_connection(); + ASSERT_TRUE(shared_state->get_connections_count() == 1); + bool success = shared_state->for_connection(tag, [shared_state](context_t& context){ + shared_state->close(context.m_connection_id); + context.m_remote_address.get_zone(); + return true; + }); + ASSERT_TRUE(success); + + ASSERT_TRUE(shared_state->get_connections_count() == 0); + constexpr auto N = 8; + tags_t tags(N); + for(auto &t: tags) + t = create_connection(); + ASSERT_TRUE(shared_state->get_connections_count() == N); + size_t index = 0; + success = shared_state->foreach_connection([&index, shared_state, &tags, &create_connection](context_t& context){ + if (!index) + for (const auto &t: tags) + shared_state->close(t); + + shared_state->close(context.m_connection_id); + context.m_remote_address.get_zone(); + ++index; + + for(auto i = 0; i < N; ++i) + create_connection(); + return true; + }); + ASSERT_TRUE(success); + ASSERT_TRUE(index == N); + ASSERT_TRUE(shared_state->get_connections_count() == N * N); + + index = 0; + success = shared_state->foreach_connection([&index, shared_state](context_t& context){ + shared_state->close(context.m_connection_id); + context.m_remote_address.get_zone(); + ++index; + return true; + }); + ASSERT_TRUE(success); + ASSERT_TRUE(index == N * N); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + + ASSERT_TRUE(shared_state->get_connections_count() == 0); + constexpr auto DELAY = 30; + constexpr auto TIMEOUT = 1; + server.get_config_shared()->set_handler(new command_handler_t(DELAY), &command_handler_t::destroy); + for (auto i = 0; i < N; ++i) { + tag = create_connection(); + ASSERT_TRUE(shared_state->get_connections_count() == 1); + success = shared_state->invoke_async(1, {}, tag, [](int, const epee::span<const uint8_t>, context_t&){}, TIMEOUT); + ASSERT_TRUE(success); + while (shared_state->sock_count == 1) { + success = shared_state->foreach_connection([&shared_state, &tag](context_t&){ + return shared_state->request_callback(tag); + }); + ASSERT_TRUE(success); + } + shared_state->close(tag); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + } + }); + + for (auto& w: workers) { + w.join(); + } + server.send_stop_signal(); + server.timed_wait_server_stop(5 * 1000); + server.deinit_server(); +} diff --git a/tests/unit_tests/epee_serialization.cpp b/tests/unit_tests/epee_serialization.cpp index cade16f0d..95a2b6ecd 100644 --- a/tests/unit_tests/epee_serialization.cpp +++ b/tests/unit_tests/epee_serialization.cpp @@ -30,6 +30,7 @@ #include <gtest/gtest.h> #include "storages/portable_storage.h" +#include "span.h" TEST(epee_binary, two_keys) { diff --git a/utils/health/README.md b/utils/health/README.md index dea46280e..8fadd3908 100644 --- a/utils/health/README.md +++ b/utils/health/README.md @@ -15,6 +15,10 @@ On the first run, the script will complain about the missing ClangBuildAnalyzer `utils/health/clang-tidy-run.sh` Performs Lint checks on the source code and stores the result in the build directory. More information on the [home page](https://clang.llvm.org/extra/clang-tidy/). +##include-what-you-use +`utils/health/clang-include-what-you-use-run.sh` +Analyses the header file hierarchy and delivers hints on how to reduce their complexity. More information on the [home page](https://include-what-you-use.org/). + ##Valgrind checks `utils/health/valgrind-tests.sh` diff --git a/utils/health/clang-include-what-you-use-run.sh b/utils/health/clang-include-what-you-use-run.sh new file mode 100755 index 000000000..655a188bd --- /dev/null +++ b/utils/health/clang-include-what-you-use-run.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Include What You Use analyses the complexity of your header hierarchy and proposes optimisations. +# User documentation: +# https://github.com/include-what-you-use/include-what-you-use/blob/master/README.md + +# Build variables +PROG="include-what-you-use" +PROG_SHORT="iwyu" +DIR_BUILD="build/clang-$PROG_SHORT" + +RESULT="$PROG_SHORT-result.txt" + +if hash "$PROG"; then + echo "Found: $PROG" +else + echo "Couldn't find: $PROG" + echo "Please run the below command to install $PROG:" + echo "sudo apt install $PROG_SHORT" + exit 1 +fi + +mkdir -p "$DIR_BUILD" && cd "$DIR_BUILD" +rm `find . -name "CMakeCache.txt"` || true + +UWYU_COMMAND="$PROG;-Xiwyu;any;-Xiwyu;iwyu;-Xiwyu;args" # Copy-pasted from the user docs. + +cmake ../.. \ +-DCMAKE_C_COMPILER=clang \ +-DCMAKE_CXX_COMPILER=clang++ \ +-DUSE_CCACHE=ON \ +-DCMAKE_C_INCLUDE_WHAT_YOU_USE="$UWYU_COMMAND" \ +-DCMAKE_CXX_INCLUDE_WHAT_YOU_USE="$UWYU_COMMAND" \ +-DBUILD_SHARED_LIBS=ON \ +-DBUILD_TESTS=ON + +make clean # Clean up to generate the full report +time make -k 2>&1 | tee "$RESULT" # Run the scan. -k means: ignore errors +#time make -k easylogging 2>&1 | tee $RESULT # Quick testing: build a single target +KPI=$(cat "$RESULT" | wc -l) +tar -cJvf "$RESULT.txz" "$RESULT" # Zip the result, because it's huge. +rm -v "$RESULT" + +echo "" +echo "Readable result stored in: $DIR_BUILD/$RESULT.gz" + +echo "$KPI" > "kpis.txt" diff --git a/utils/health/valgrind-tests.sh b/utils/health/valgrind-tests.sh new file mode 100755 index 000000000..9f5e7e7c0 --- /dev/null +++ b/utils/health/valgrind-tests.sh @@ -0,0 +1,161 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This script is able to run valgrind's callgrind, cachegrind and memcheck for a given set of executables. +# It expects ONE PARAMETER, which points to a file with paths to executables and their arguments, written line by line. + +if [ "$#" -ne 1 ]; then + echo "Please provide an argument, which points to a file with paths to executables and their arguments, written line by line. For example:" + echo "" + echo "ls -l -h" + echo "build/tests/unit_tests/unit_tests" + exit 1 +fi + +FILE_IN="$1" +DIR_OUT="build/valgrind-output" # Using build as the base output directory, as it's ignored in .gitignore + +function is_file_or_exit { + FILE="${1}" + if [ -f $FILE ]; then + echo "The input file $FILE exists. Can proceed." + else + echo "The input file $FILE doesn't exist." + exit 1 + fi + return 0 +} + +function is_tool_or_exit { + TOOL="${1}" + if $(hash ${TOOL}); then + echo "${TOOL} is installed. Can proceed." + else + echo "Please install ${TOOL} to continue." + exit 1 + fi + return 0 +} + +function get_tool_out_file_base { + EXE="${1}" + TOOL="${2}" + + EXE_NAME=$(basename $EXE) + local retval="${DIR_OUT}/${EXE_NAME}-${TOOL}" + echo "$retval" +} + +function get_tool_out_file { + EXE="${1}" + TOOL="${2}" + + FILE_OUT_BASE=$(get_tool_out_file_base ${EXE} ${TOOL}) + local retval="--${TOOL}-out-file=${FILE_OUT_BASE}.out" + echo "$retval" +} + +function run_valgrind_4_executable { + EXE="${1}" + ARGS="${2}" + TOOL="${3}" + EXTRA_OPTS="${4}" + FILE_OUT_TOOL="${5}" + FILE_OUT_BASE=$(get_tool_out_file_base ${EXE} ${TOOL}) + + echo "Runnig '${TOOL}' for '${EXE}' with args '${ARGS}'" + echo "EXTRA_OPTS = ${EXTRA_OPTS}" + echo "FILE_OUT_TOOL = ${FILE_OUT_TOOL}" + if ! valgrind --tool=${TOOL} ${FILE_OUT_TOOL} --log-file="${FILE_OUT_BASE}.log" ${EXTRA_OPTS} ${EXE} ${ARGS}; then + echo "FAILED in runnig ${TOOL} for ${EXE} !" + fi +} + +function run_valgrind_4_executable_callgrind { + EXE="${1}" + ARGS="${2}" + TOOL="callgrind" + EXTRA_OPTS="--dump-instr=yes --simulate-cache=yes --collect-jumps=yes" + FILE_OUT_TOOL=$(get_tool_out_file ${EXE} ${TOOL}) + + run_valgrind_4_executable ${EXE} "${ARGS}" ${TOOL} "${EXTRA_OPTS}" ${FILE_OUT_TOOL} +} + +function run_valgrind_4_executable_cachegrind { + EXE="${1}" + ARGS="${2}" + TOOL="cachegrind" + EXTRA_OPTS="" + FILE_OUT_TOOL=$(get_tool_out_file ${EXE} ${TOOL}) + + run_valgrind_4_executable ${EXE} "${ARGS}" ${TOOL} "${EXTRA_OPTS}" ${FILE_OUT_TOOL} +} + +function run_valgrind_4_executable_memcheck { + EXE="${1}" + ARGS="${2}" + TOOL="memcheck" + #EXTRA_OPTS="--leak-check=yes" # Minimalistic + EXTRA_OPTS="--leak-check=full --show-leak-kinds=all --track-origins=yes" + FILE_OUT_TOOL="" # memcheck has no special out file, only the log + + run_valgrind_4_executable ${EXE} "${ARGS}" ${TOOL} "${EXTRA_OPTS}" ${FILE_OUT_TOOL} +} + +function run_valgrind_4_executable_all { + EXE_ARGS_ARR=(${1}) + EXE=${EXE_ARGS_ARR[0]} # First element of the array + ARGS=${EXE_ARGS_ARR[@]:1} # Every next element + + #EXE="ls" # A quick check of the happy path + #EXE="nothere" # A quick check of error handling - no such executable + #EXE=/bin/false # A quick check of error handling - executable returned != 0 + + run_valgrind_4_executable_memcheck ${EXE} "${ARGS}" + run_valgrind_4_executable_cachegrind ${EXE} "${ARGS}" + run_valgrind_4_executable_callgrind ${EXE} "${ARGS}" +} + +is_tool_or_exit valgrind +is_file_or_exit "$FILE_IN" +echo "All OK." +echo "Will perform checks for the following executables and their arguments:" +while IFS= read -r line; do + echo "$line" +done < "$FILE_IN" + +mkdir -p "$DIR_OUT" +while IFS= read -r line; do + echo "$line" + run_valgrind_4_executable_all "$line" +done < "$FILE_IN" + +echo "Done. All data saved in ${DIR_OUT}" + diff --git a/utils/python-rpc/console.py b/utils/python-rpc/console.py index 57a04528b..c94e07eab 100755 --- a/utils/python-rpc/console.py +++ b/utils/python-rpc/console.py @@ -29,7 +29,7 @@ for n in range(1, len(sys.argv)): raise Exception(USAGE) if port <= 0 or port > 65535: raise Exception(USAGE) - except Exception, e: + except Exception as e: print('Error: ' + str(e)) raise Exception(USAGE) @@ -49,7 +49,7 @@ for n in range(1, len(sys.argv)): } try: res = rpc.send_json_rpc_request(get_version) - except Exception, e: + except Exception as e: raise Exception('Failed to call version RPC: ' + str(e)) if 'version' not in res: |