diff options
42 files changed, 752 insertions, 97 deletions
@@ -1,4 +1,4 @@ -Copyright (c) 2014-2020, The Monero Project +Copyright (c) 2014-2021, The Monero Project All rights reserved. diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index ad561c5b6..fce6d4b7e 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -72,7 +72,8 @@ namespace levin #define LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED 0 -#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000 //100MB by default +#define LEVIN_INITIAL_MAX_PACKET_SIZE 256*1024 // 256 KiB before handshake +#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000 //100MB by default after handshake #define LEVIN_PACKET_REQUEST 0x00000001 #define LEVIN_PACKET_RESPONSE 0x00000002 diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 8cb2be3e1..ddde701ee 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -84,7 +84,8 @@ class async_protocol_handler_config public: typedef t_connection_context connection_context; - uint64_t m_max_packet_size; + uint64_t m_initial_max_packet_size; + uint64_t m_max_packet_size; uint64_t m_invoke_timeout; int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); @@ -105,7 +106,7 @@ public: size_t get_in_connections_count(); void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL); - async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_initial_max_packet_size(LEVIN_INITIAL_MAX_PACKET_SIZE), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) {} ~async_protocol_handler_config() { set_handler(NULL, NULL); } void del_out_connections(size_t count); @@ -162,6 +163,7 @@ public: net_utils::i_service_endpoint* m_pservice_endpoint; config_type& m_config; t_connection_context& m_connection_context; + std::atomic<uint64_t> m_max_packet_size; net_utils::buffer m_cache_in_buffer; stream_state m_state; @@ -289,7 +291,8 @@ public: m_current_head(bucket_head2()), m_pservice_endpoint(psnd_hndlr), m_config(config), - m_connection_context(conn_context), + m_connection_context(conn_context), + m_max_packet_size(config.m_initial_max_packet_size), m_cache_in_buffer(4 * 1024), m_state(stream_state_head) { @@ -399,13 +402,14 @@ public: } // these should never fail, but do runtime check for safety - CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()"); - CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()"); + const uint64_t max_packet_size = m_max_packet_size; + CHECK_AND_ASSERT_MES(max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()"); + CHECK_AND_ASSERT_MES(max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()"); // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public - if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size()) + if(cb > max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size()) { - MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size + MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << max_packet_size << ", packet received " << m_cache_in_buffer.size() + cb << ", connection will be closed."); return false; @@ -430,7 +434,7 @@ public: //async call scenario boost::shared_ptr<invoke_response_handler_base> response_handler = m_invoke_response_handlers.front(); response_handler->reset_timer(); - MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb); + MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb << ", current total " << m_cache_in_buffer.size() << "/" << m_current_head.m_cb << " (" << (100.0f * m_cache_in_buffer.size() / (m_current_head.m_cb ? m_current_head.m_cb : 1)) << "%)"); } } break; @@ -465,6 +469,14 @@ public: temp = std::move(m_fragment_buffer); m_fragment_buffer.clear(); std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2)); + const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command); + if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes)) + { + MERROR(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes) + << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command + << ", connection will be closed."); + return false; + } buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)}; } @@ -519,6 +531,10 @@ public: m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context ); + // peer_id remains unset if dropped + if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) + m_max_packet_size = m_config.m_max_packet_size; + bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); head.m_return_code = SWAP32LE(return_code); @@ -576,10 +592,11 @@ public: m_cache_in_buffer.erase(sizeof(bucket_head2)); m_state = stream_state_body; m_oponent_protocol_ver = m_current_head.m_protocol_version; - if(m_current_head.m_cb > m_config.m_max_packet_size) + const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command); + if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes)) { - LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size - << ", packet header received " << m_current_head.m_cb + LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes) + << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command << ", connection will be closed."); return false; } @@ -633,6 +650,9 @@ public: boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock); + if (command == m_connection_context.handshake_command()) + m_max_packet_size = m_config.m_max_packet_size; + if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); @@ -674,6 +694,9 @@ public: boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); + if (command == m_connection_context.handshake_command()) + m_max_packet_size = m_config.m_max_packet_size; + if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index c4cb91130..c615b20e6 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -98,7 +98,12 @@ namespace epee return false; } - return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body)); + static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = { + 65536 * 3, // objects + 65536 * 3, // fields + 65536 * 3, // strings + }; + return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits); } template<class t_request, class t_response, class t_transport> diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 95f0bb410..802e16c1b 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -52,6 +52,11 @@ namespace snprintf(buf, sizeof(buf), "command-%u", command); return on_levin_traffic(context, initiator, sent, error, bytes, buf); } + static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { + 8192, // objects + 16384, // fields + 16384, // strings + }; } namespace epee @@ -77,7 +82,7 @@ namespace epee return false; } serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv)) + if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) { LOG_ERROR("Failed to load_from_binary on command " << command); return false; @@ -124,7 +129,7 @@ namespace epee return false; } typename serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff_to_recv)) + if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits)) { on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); @@ -155,7 +160,7 @@ namespace epee return false; } serialization::portable_storage stg_ret; - if(!stg_ret.load_from_binary(buff)) + if(!stg_ret.load_from_binary(buff, &default_levin_limits)) { on_levin_traffic(context, true, false, true, buff.size(), command); LOG_ERROR("Failed to load_from_binary on command " << command); @@ -205,7 +210,7 @@ namespace epee int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff)) + if(!strg.load_from_binary(in_buff, &default_levin_limits)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in command " << command); @@ -239,7 +244,7 @@ namespace epee int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context) { serialization::portable_storage strg; - if(!strg.load_from_binary(in_buff)) + if(!strg.load_from_binary(in_buff, &default_levin_limits)) { on_levin_traffic(context, false, false, true, in_buff.size(), command); LOG_ERROR("Failed to load_from_binary in notify " << command); diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index 589e6ad63..f77e89cb6 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -54,6 +54,13 @@ namespace epee typedef epee::serialization::harray harray; typedef storage_entry meta_entry; + struct limits_t + { + size_t n_objects; + size_t n_fields; + size_t n_strings; // not counting field names + }; + portable_storage(){} virtual ~portable_storage(){} hsection open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist = false); @@ -84,8 +91,8 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target); - bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); } + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); + bool load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); } template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -134,7 +141,7 @@ namespace epee return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format } inline - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source) + bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) { m_root.m_entries.clear(); if(source.size() < sizeof(storage_block_header)) @@ -157,6 +164,8 @@ namespace epee } TRY_ENTRY(); throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); + if (limits) + buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); buf_reader.read(m_root); return true;//TODO: CATCH_ENTRY("portable_storage::load_from_binary", false); @@ -266,6 +275,7 @@ namespace epee static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value"); TRY_ENTRY(); CHECK_AND_ASSERT(psection, nullptr); + CHECK_AND_ASSERT(!pentry_name.empty(), nullptr); auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry)); return &ins_res.first->second; CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr); diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index eb0eed235..9e7b6ec34 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -29,6 +29,7 @@ #pragma once #include "misc_language.h" +#include "misc_log_ex.h" #include "portable_storage_base.h" #include "portable_storage_bin_utils.h" @@ -37,7 +38,6 @@ #else #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL 100 #endif -#define EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL 65536 namespace epee { @@ -46,21 +46,20 @@ namespace epee template<typename T> struct ps_min_bytes { static constexpr const size_t strict = 4096; // actual low bound - static constexpr const size_t rough = 4096; // when we want to be stricter for DoS prevention }; - template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8, rough = 8; }; - template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8, rough = 8; }; - template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4, rough = 4; }; - template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4, rough = 4; }; - template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2, rough = 2; }; - template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2, rough = 2; }; - template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1, rough = 1; }; - template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1, rough = 1; }; - template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8, rough = 8; }; - template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1, rough = 1; }; - template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2, rough = 16; }; - template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1, rough = 256; }; - template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1, rough = 128; }; + template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; }; + template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; }; + template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; }; + template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; }; + template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; }; + template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; }; struct throwable_buffer_reader { @@ -83,6 +82,7 @@ namespace epee void read(array_entry &ae); template<class t_type> size_t min_bytes() const; + void set_limits(size_t objects, size_t fields, size_t strings); private: struct recursuion_limitation_guard { @@ -104,6 +104,12 @@ namespace epee size_t m_count; size_t m_recursion_count; size_t m_objects; + size_t m_fields; + size_t m_strings; + + size_t max_objects; + size_t max_fields; + size_t max_strings; }; inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz) @@ -116,6 +122,11 @@ namespace epee m_count = sz; m_recursion_count = 0; m_objects = 0; + m_fields = 0; + m_strings = 0; + max_objects = std::numeric_limits<size_t>::max(); + max_fields = std::numeric_limits<size_t>::max(); + max_strings = std::numeric_limits<size_t>::max(); } inline void throwable_buffer_reader::read(void* target, size_t count) @@ -132,6 +143,7 @@ namespace epee RECURSION_LIMITATION(); uint8_t name_len = 0; read(name_len); + CHECK_AND_ASSERT_THROW_MES(name_len > 0, "Section name is missing"); sce_name.resize(name_len); read((void*)sce_name.data(), name_len); } @@ -163,6 +175,16 @@ namespace epee array_entry_t<type_name> sa; size_t size = read_varint(); CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed"); + if (std::is_same<type_name, section>()) + { + CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects"); + m_objects += size; + } + else if (std::is_same<type_name, std::string>()) + { + CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings"); + m_strings += size; + } sa.reserve(size); //TODO: add some optimization here later @@ -229,6 +251,8 @@ namespace epee inline storage_entry throwable_buffer_reader::read_se<std::string>() { RECURSION_LIMITATION(); + CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings"); + m_strings += 1; return storage_entry(read<std::string>()); } @@ -237,6 +261,8 @@ namespace epee inline storage_entry throwable_buffer_reader::read_se<section>() { RECURSION_LIMITATION(); + CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects"); + ++m_objects; section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio storage_entry se(std::move(s)); section& section_entry = boost::get<section>(se); @@ -288,14 +314,16 @@ namespace epee RECURSION_LIMITATION(); sec.m_entries.clear(); size_t count = read_varint(); - CHECK_AND_ASSERT_THROW_MES(count < EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL - m_objects, "Too many objects"); - m_objects += count; + CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields"); + m_fields += count; while(count--) { //read section name string std::string sec_name; read_sec_name(sec_name); - sec.m_entries.emplace(std::move(sec_name), load_storage_entry()); + const auto insert_loc = sec.m_entries.lower_bound(sec_name); + CHECK_AND_ASSERT_THROW_MES(insert_loc == sec.m_entries.end() || insert_loc->first != sec_name, "duplicate key: " << sec_name); + sec.m_entries.emplace_hint(insert_loc, std::move(sec_name), load_storage_entry()); } } inline @@ -316,5 +344,12 @@ namespace epee RECURSION_LIMITATION(); CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported"); } + inline + void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings) + { + max_objects = objects; + max_fields = fields; + max_strings = strings; + } } } diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 39f900c8d..16dd565ec 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -85,10 +85,10 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template<class t_struct> - bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff) + bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL) { portable_storage ps; - bool rs = ps.load_from_binary(binary_buff); + bool rs = ps.load_from_binary(binary_buff, limits); if(!rs) return false; diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 137497e19..49a7be185 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -211,6 +211,7 @@ namespace epee for(const section_pair& se: sec.m_entries) { CHECK_AND_ASSERT_THROW_MES(se.first.size() < std::numeric_limits<uint8_t>::max(), "storage_entry_name is too long: " << se.first.size() << ", val: " << se.first); + CHECK_AND_ASSERT_THROW_MES(!se.first.empty(), "storage_entry_name is empty"); uint8_t len = static_cast<uint8_t>(se.first.size()); strm.write((const char*)&len, sizeof(len)); strm.write(se.first.data(), size_t(len)); diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index bcf499963..1047d1696 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -6,6 +6,7 @@ #include <boost/thread/lock_guard.hpp> #include <boost/algorithm/string.hpp> +static bool same_as_last_line(const std::string&); static void install_line_handler(); static void remove_line_handler(); @@ -175,8 +176,11 @@ static void handle_line(char* line) boost::trim_right(test_line); if(!test_line.empty()) { - add_history(test_line.c_str()); - history_set_pos(history_length); + if (!same_as_last_line(test_line)) + { + add_history(test_line.c_str()); + history_set_pos(history_length); + } if (test_line == "exit" || test_line == "q") exit = true; } @@ -192,6 +196,16 @@ static void handle_line(char* line) return; } +// same_as_last_line returns true, if the last line in the history is +// equal to test_line. +static bool same_as_last_line(const std::string& test_line) +{ + // Note that state->offset == state->length, when a new line was entered. + HISTORY_STATE* state = history_get_history_state(); + return state->length > 0 + && test_line.compare(state->entries[state->length-1]->line) == 0; +} + static char* completion_matches(const char* text, int state) { static size_t list_index; diff --git a/contrib/fuzz_testing/fuzz.sh b/contrib/fuzz_testing/fuzz.sh index efd43c231..5c88c3727 100755 --- a/contrib/fuzz_testing/fuzz.sh +++ b/contrib/fuzz_testing/fuzz.sh @@ -14,8 +14,8 @@ then exit 1 fi case "$type" in - block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;; - *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;; + block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8) ;; + *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8"; exit 1 ;; esac if test -d "fuzz-out/$type" diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index 113fd9d86..5286256c7 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -38,6 +38,7 @@ endif() set(cryptonote_basic_sources account.cpp + connection_context.cpp cryptonote_basic_impl.cpp cryptonote_format_utils.cpp difficulty.cpp diff --git a/src/cryptonote_basic/connection_context.cpp b/src/cryptonote_basic/connection_context.cpp new file mode 100644 index 000000000..a0b8ca1f1 --- /dev/null +++ b/src/cryptonote_basic/connection_context.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "connection_context.h" + +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "p2p/p2p_protocol_defs.h" + +namespace cryptonote +{ + std::size_t cryptonote_connection_context::get_max_bytes(const int command) noexcept + { + switch (command) + { + case nodetool::COMMAND_HANDSHAKE_T<cryptonote::CORE_SYNC_DATA>::ID: + return 65536; + case nodetool::COMMAND_TIMED_SYNC_T<cryptonote::CORE_SYNC_DATA>::ID: + return 65536; + case nodetool::COMMAND_PING::ID: + return 4096; + case nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::ID: + return 4096; + case cryptonote::NOTIFY_NEW_BLOCK::ID: + return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though) + case cryptonote::NOTIFY_NEW_TRANSACTIONS::ID: + return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though) + case cryptonote::NOTIFY_REQUEST_GET_OBJECTS::ID: + return 1024 * 1024 * 2; // 2 MB + case cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::ID: + return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though) + case cryptonote::NOTIFY_REQUEST_CHAIN::ID: + return 512 * 1024; // 512 kB + case cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::ID: + return 1024 * 1024 * 4; // 4 MB + case cryptonote::NOTIFY_NEW_FLUFFY_BLOCK::ID: + return 1024 * 1024 * 4; // 4 MB, but it does not includes transaction data + case cryptonote::NOTIFY_REQUEST_FLUFFY_MISSING_TX::ID: + return 1024 * 1024; // 1 MB + case cryptonote::NOTIFY_GET_TXPOOL_COMPLEMENT::ID: + return 1024 * 1024 * 4; // 4 MB + default: + break; + }; + return std::numeric_limits<size_t>::max(); + } +} // cryptonote diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 9e012f8f5..a7d688300 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -31,6 +31,7 @@ #pragma once #include <unordered_set> #include <atomic> +#include <algorithm> #include <boost/date_time/posix_time/posix_time.hpp> #include "net/net_utils_base.h" #include "copyable_atomic.h" @@ -38,7 +39,6 @@ namespace cryptonote { - struct cryptonote_connection_context: public epee::net_utils::connection_context_base { cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0), @@ -55,6 +55,12 @@ namespace cryptonote state_normal }; + static constexpr int handshake_command() noexcept { return 1001; } + bool handshake_complete() const noexcept { return m_state != state_before_handshake; } + + //! \return Maximum number of bytes permissible for `command`. + static size_t get_max_bytes(int command) noexcept; + state m_state; std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects; std::unordered_set<crypto::hash> m_requested_objects; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 5b3b0527b..8ec624254 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2758,32 +2758,44 @@ void Blockchain::flush_invalid_blocks() m_invalid_blocks.clear(); } //------------------------------------------------------------------ -bool Blockchain::have_block(const crypto::hash& id) const +bool Blockchain::have_block_unlocked(const crypto::hash& id, int *where) const { + // WARNING: this function does not take m_blockchain_lock, and thus should only call read only + // m_db functions which do not depend on one another (ie, no getheight + gethash(height-1), as + // well as not accessing class members, even read only (ie, m_invalid_blocks). The caller must + // lock if it is otherwise needed. LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_db->block_exists(id)) { LOG_PRINT_L2("block " << id << " found in main chain"); + if (where) *where = HAVE_BLOCK_MAIN_CHAIN; return true; } if(m_db->get_alt_block(id, NULL, NULL)) { LOG_PRINT_L2("block " << id << " found in alternative chains"); + if (where) *where = HAVE_BLOCK_ALT_CHAIN; return true; } if(m_invalid_blocks.count(id)) { LOG_PRINT_L2("block " << id << " found in m_invalid_blocks"); + if (where) *where = HAVE_BLOCK_INVALID; return true; } return false; } //------------------------------------------------------------------ +bool Blockchain::have_block(const crypto::hash& id, int *where) const +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return have_block_unlocked(id, where); +} +//------------------------------------------------------------------ bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc, bool notify/* = true*/) { LOG_PRINT_L3("Blockchain::" << __func__); @@ -4716,6 +4728,8 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector CHECK_AND_ASSERT_MES(weights.empty() || weights.size() == hashes.size(), 0, "Unexpected weights size"); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + // easy case: height >= hashes if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP) return hashes.size(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 07238b719..5291f1338 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -377,10 +377,12 @@ namespace cryptonote * for a block with the given hash * * @param id the hash to search for + * @param where the type of block, if non NULL * * @return true if the block is known, else false */ - bool have_block(const crypto::hash& id) const; + bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const; + bool have_block(const crypto::hash& id, int *where = NULL) const; /** * @brief gets the total number of transactions on the main chain diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 507c47a51..12125fb9d 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1625,9 +1625,14 @@ namespace cryptonote return m_mempool.get_transactions_count(include_sensitive_txes); } //----------------------------------------------------------------------------------------------- - bool core::have_block(const crypto::hash& id) const + bool core::have_block_unlocked(const crypto::hash& id, int *where) const { - return m_blockchain_storage.have_block(id); + return m_blockchain_storage.have_block_unlocked(id, where); + } + //----------------------------------------------------------------------------------------------- + bool core::have_block(const crypto::hash& id, int *where) const + { + return m_blockchain_storage.have_block(id, where); } //----------------------------------------------------------------------------------------------- bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 24d0f9e76..8891540a9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -55,6 +55,8 @@ PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) +enum { HAVE_BLOCK_MAIN_CHAIN, HAVE_BLOCK_ALT_CHAIN, HAVE_BLOCK_INVALID }; + namespace cryptonote { struct test_options { @@ -543,7 +545,8 @@ namespace cryptonote * * @note see Blockchain::have_block */ - bool have_block(const crypto::hash& id) const; + bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const; + bool have_block(const crypto::hash& id, int *where = NULL) const; /** * @copydoc Blockchain::get_short_chain_history diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 22e87465f..b57fc0f0f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -137,7 +137,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count); --context.m_callback_request_count; - if(context.m_state == cryptonote_connection_context::state_synchronizing) + if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time) { NOTIFY_REQUEST_CHAIN::request r = {}; context.m_needed_objects.clear(); @@ -549,6 +549,7 @@ namespace cryptonote } std::vector<tx_blob_entry> have_tx; + have_tx.reserve(new_block.tx_hashes.size()); // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -557,6 +558,7 @@ namespace cryptonote // Also, remember to pepper some whitespace changes around to bother // moneromooo ... only because I <3 him. std::vector<uint64_t> need_tx_indices; + need_tx_indices.reserve(new_block.tx_hashes.size()); transaction tx; crypto::hash tx_hash; @@ -829,6 +831,7 @@ namespace cryptonote } std::vector<crypto::hash> txids; + txids.reserve(b.tx_hashes.size()); NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response; fluffy_response.b.block = t_serializable_object_to_blob(b); fluffy_response.current_blockchain_height = arg.current_blockchain_height; @@ -2189,6 +2192,7 @@ skip: if (span.second > 0) { is_next = true; + req.blocks.reserve(hashes.size()); for (const auto &hash: hashes) { req.blocks.push_back(hash); @@ -2247,6 +2251,7 @@ skip: if (span.second > 0) { is_next = true; + req.blocks.reserve(hashes.size()); for (const auto &hash: hashes) { req.blocks.push_back(hash); @@ -2280,6 +2285,7 @@ skip: return false; } + req.blocks.reserve(req.blocks.size() + span.second); for (size_t n = 0; n < span.second; ++n) { req.blocks.push_back(context.m_needed_objects[n].first); @@ -2570,17 +2576,6 @@ skip: return 1; } - std::unordered_set<crypto::hash> hashes; - for (const auto &h: arg.m_block_ids) - { - if (!hashes.insert(h).second) - { - LOG_ERROR_CCONTEXT("sent duplicate block, dropping connection"); - drop_connection(context, true, false); - return 1; - } - } - uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids, arg.m_block_weights); if (n_use_blocks == 0 || n_use_blocks + HASH_OF_HASHES_STEP <= arg.m_block_ids.size()) { @@ -2590,20 +2585,77 @@ skip: } context.m_needed_objects.clear(); + context.m_needed_objects.reserve(arg.m_block_ids.size()); uint64_t added = 0; std::unordered_set<crypto::hash> blocks_found; + bool first = true; + bool expect_unknown = false; for (size_t i = 0; i < arg.m_block_ids.size(); ++i) { if (!blocks_found.insert(arg.m_block_ids[i]).second) { LOG_ERROR_CCONTEXT("Duplicate blocks in chain entry response, dropping connection"); - drop_connection(context, true, false); + drop_connection_with_score(context, 5, false); return 1; } + int where; + const bool have_block = m_core.have_block_unlocked(arg.m_block_ids[i], &where); + if (first) + { + if (!have_block && !m_block_queue.requested(arg.m_block_ids[i]) && !m_block_queue.have(arg.m_block_ids[i])) + { + LOG_ERROR_CCONTEXT("First block hash is unknown, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + if (!have_block) + expect_unknown = true; + } + if (!first) + { + // after the first, blocks may be known or unknown, but if they are known, + // they should be at the same height if on the main chain + if (have_block) + { + switch (where) + { + default: + case HAVE_BLOCK_INVALID: + LOG_ERROR_CCONTEXT("Block is invalid or known without known type, dropping connection"); + drop_connection(context, true, false); + return 1; + case HAVE_BLOCK_MAIN_CHAIN: + if (expect_unknown) + { + LOG_ERROR_CCONTEXT("Block is on the main chain, but we did not expect a known block, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + if (m_core.get_block_id_by_height(arg.start_height + i) != arg.m_block_ids[i]) + { + LOG_ERROR_CCONTEXT("Block is on the main chain, but not at the expected height, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + break; + case HAVE_BLOCK_ALT_CHAIN: + if (expect_unknown) + { + LOG_ERROR_CCONTEXT("Block is on the main chain, but we did not expect a known block, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + break; + } + } + else + expect_unknown = true; + } const uint64_t block_weight = arg.m_block_weights.empty() ? 0 : arg.m_block_weights[i]; context.m_needed_objects.push_back(std::make_pair(arg.m_block_ids[i], block_weight)); if (++added == n_use_blocks) break; + first = false; } context.m_last_response_height -= arg.m_block_ids.size() - n_use_blocks; @@ -2634,6 +2686,7 @@ skip: std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { + // peer_id also filters out connections before handshake if (peer_id && exclude_context.m_connection_id != context.m_connection_id && context.m_remote_address.get_zone() == epee::net_utils::zone::public_) { if(m_core.fluffy_blocks_enabled() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS)) @@ -2794,12 +2847,15 @@ skip: epee::string_tools::to_string_hex(context.m_pruning_seed) << "), score " << score << ", flush_all_spans " << flush_all_spans); - if (score > 0) - m_p2p->add_host_fail(context.m_remote_address, score); - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + // copy since dropping the connection will invalidate the context, and thus the address + const auto remote_address = context.m_remote_address; + m_p2p->drop_connection(context); + + if (score > 0) + m_p2p->add_host_fail(remote_address, score); } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 21363972d..1e9f3e399 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -51,14 +51,6 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.p2p.tx" -namespace -{ - int get_command_from_message(const epee::byte_slice &msg) - { - return msg.size() >= sizeof(epee::levin::bucket_head2) ? SWAP32LE(((epee::levin::bucket_head2*)msg.data())->m_command) : 0; - } -} - namespace cryptonote { namespace levin @@ -212,7 +204,7 @@ namespace levin { const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); p2p.for_connection(destination, [&blob](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob)); + on_levin_traffic(context, true, true, false, blob.size(), NOTIFY_NEW_TRANSACTIONS::ID); return true; }); return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); @@ -443,7 +435,7 @@ namespace levin zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context) { // When i2p/tor, only fluff to outbound connections - if (source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) + if (context.handshake_complete() && source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) { if (context.fluff_txs.empty()) context.flush_time = now + (context.m_is_income ? in_duration() : out_duration()); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 04feb55fd..5a7d4dd4e 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -2275,6 +2275,7 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; + tools::success_msg_writer() << "Remote Host Peer_ID State Prune_Seed Height DL kB/s, Queued Blocks / MB"; for (const auto &p: res.peers) { std::string address = epee::string_tools::pad_string(p.info.address, 24); diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 9fba5d636..59a6e5091 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -139,6 +139,7 @@ namespace nodetool typedef COMMAND_HANDSHAKE_T<typename t_payload_net_handler::payload_type> COMMAND_HANDSHAKE; typedef COMMAND_TIMED_SYNC_T<typename t_payload_net_handler::payload_type> COMMAND_TIMED_SYNC; + static_assert(p2p_connection_context::handshake_command() == COMMAND_HANDSHAKE::ID, "invalid handshake command id"); typedef epee::net_utils::boosted_tcp_server<epee::levin::async_protocol_handler<p2p_connection_context>> net_server; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index bf053f0f2..b8bd7b2a7 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1217,8 +1217,9 @@ namespace nodetool if(!handle_remote_peerlist(rsp.local_peerlist_new, context)) { LOG_WARNING_CC(context, "COMMAND_TIMED_SYNC: failed to handle_remote_peerlist(...), closing connection."); + const auto remote_address = context.m_remote_address; m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); - add_host_fail(context.m_remote_address); + add_host_fail(remote_address); } if(!context.m_is_income) m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash); @@ -1382,7 +1383,7 @@ namespace nodetool if(just_take_peerlist) { zone.m_net_server.get_config_object().close(con->m_connection_id); - LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); + MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -1444,7 +1445,7 @@ namespace nodetool zone.m_net_server.get_config_object().close(con->m_connection_id); - LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); + MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -2149,6 +2150,7 @@ namespace nodetool LOG_DEBUG_CC(context, "REMOTE PEERLIST: remote peerlist size=" << peerlist_.size()); LOG_TRACE_CC(context, "REMOTE PEERLIST: " << ENDL << print_peerlist_to_string(peerlist_)); + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_, [this](const peerlist_entry &pe) { return !is_addr_recently_failed(pe.adr) && is_remote_host_allowed(pe.adr); }); @@ -2471,12 +2473,14 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_handshake(int command, typename COMMAND_HANDSHAKE::request& arg, typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { + // copy since dropping the connection will invalidate the context, and thus the address + const auto remote_address = context.m_remote_address; + if(arg.node_data.network_id != m_network_id) { - LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); - add_host_fail(context.m_remote_address); + add_host_fail(remote_address); return 1; } @@ -2484,7 +2488,7 @@ namespace nodetool { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came not from incoming connection"); drop_connection(context); - add_host_fail(context.m_remote_address); + add_host_fail(remote_address); return 1; } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 5b2043de6..db228dd94 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -452,7 +452,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); - res.target_height = m_core.get_target_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase @@ -2913,7 +2913,7 @@ namespace cryptonote crypto::hash top_hash; m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height - res.target_height = m_core.get_target_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 0635520c6..dfd6adf3a 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -4765,9 +4765,14 @@ bool simple_wallet::try_connect_to_daemon(bool silent, uint32_t* version) if (!m_wallet->check_connection(version)) { if (!silent) - fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " << - tr("Daemon either is not started or wrong port was passed. " - "Please make sure daemon is running or change the daemon address using the 'set_daemon' command."); + { + if (m_wallet->is_offline()) + fail_msg_writer() << tr("wallet failed to connect to daemon, because it is set to offline mode"); + else + fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " << + tr("Daemon either is not started or wrong port was passed. " + "Please make sure daemon is running or change the daemon address using the 'set_daemon' command."); + } return false; } if (!m_allow_mismatched_daemon_version && ((*version >> 16) != CORE_RPC_VERSION_MAJOR)) @@ -9298,7 +9303,7 @@ bool simple_wallet::run() refresh_main(0, ResetNone, true); - m_auto_refresh_enabled = m_wallet->auto_refresh(); + m_auto_refresh_enabled = !m_wallet->is_offline() && m_wallet->auto_refresh(); m_idle_thread = boost::thread([&]{wallet_idle_thread();}); message_writer(console_color_green, false) << "Background refresh thread started"; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 68f03db72..2e455c40c 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1549,6 +1549,7 @@ private: void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); void enable_dns(bool enable) { m_use_dns = enable; } void set_offline(bool offline = true); + bool is_offline() const { return m_offline; } uint64_t credits() const { return m_rpc_payment_state.credits; } void credit_report(uint64_t &expected_spent, uint64_t &discrepancy) const { expected_spent = m_rpc_payment_state.expected_spent; discrepancy = m_rpc_payment_state.discrepancy; } diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index 8095f03e3..09be8758b 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -245,12 +245,17 @@ bool tests::proxy_core::init(const boost::program_options::variables_map& /*vm*/ return true; } -bool tests::proxy_core::have_block(const crypto::hash& id) { +bool tests::proxy_core::have_block_unlocked(const crypto::hash& id, int *where) { if (m_hash2blkidx.end() == m_hash2blkidx.find(id)) return false; + if (where) *where = HAVE_BLOCK_MAIN_CHAIN; return true; } +bool tests::proxy_core::have_block(const crypto::hash& id, int *where) { + return have_block_unlocked(id, where); +} + void tests::proxy_core::build_short_history(std::list<crypto::hash> &m_history, const crypto::hash &m_start) { m_history.push_front(get_block_hash(m_genesis)); /*std::unordered_map<crypto::hash, tests::block_index>::const_iterator cit = m_hash2blkidx.find(m_lastblk); diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index ebc3a89c2..94f148e8c 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -74,7 +74,8 @@ namespace tests bool init(const boost::program_options::variables_map& vm); bool deinit(){return true;} bool get_short_chain_history(std::list<crypto::hash>& ids); - bool have_block(const crypto::hash& id); + bool have_block(const crypto::hash& id, int *where = NULL); + bool have_block_unlocked(const crypto::hash& id, int *where = NULL); void get_blockchain_top(uint64_t& height, crypto::hash& top_id); bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, cryptonote::relay_method tx_relay, bool relayed); bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, cryptonote::relay_method tx_relay, bool relayed); diff --git a/tests/data/fuzz/utf8/UTF8_1 b/tests/data/fuzz/utf8/UTF8_1 new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/data/fuzz/utf8/UTF8_1 diff --git a/tests/data/fuzz/utf8/UTF8_2 b/tests/data/fuzz/utf8/UTF8_2 Binary files differnew file mode 100644 index 000000000..bb6982c4f --- /dev/null +++ b/tests/data/fuzz/utf8/UTF8_2 diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index a599f86f8..0cf1740ad 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -218,3 +218,13 @@ set_property(TARGET tx-extra_fuzz_tests PROPERTY FOLDER "tests") +monero_add_minimal_executable(utf8_fuzz_tests utf8.cpp fuzzer.cpp) +target_link_libraries(utf8_fuzz_tests + PRIVATE + common + epee + ${Boost_THREAD_LIBRARY} + ${Boost_CHRONO_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES} + $ENV{LIB_FUZZING_ENGINE}) diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index b090c350b..78b7b6863 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -52,6 +52,9 @@ namespace struct test_levin_connection_context : public epee::net_utils::connection_context_base { + static constexpr int handshake_command() noexcept { return 1001; } + static constexpr bool handshake_complete() noexcept { return true; } + size_t get_max_bytes(int command) const { return LEVIN_DEFAULT_MAX_PACKET_SIZE; } }; typedef epee::levin::async_protocol_handler_config<test_levin_connection_context> test_levin_protocol_handler_config; diff --git a/tests/fuzz/utf8.cpp b/tests/fuzz/utf8.cpp new file mode 100644 index 000000000..bf304a351 --- /dev/null +++ b/tests/fuzz/utf8.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2017-2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "file_io_utils.h" +#include "common/utf8.h" +#include "fuzzer.h" + +BEGIN_INIT_SIMPLE_FUZZER() +END_INIT_SIMPLE_FUZZER() + +BEGIN_SIMPLE_FUZZER() + tools::utf8canonical(std::string((const char*)buf, len), [](wint_t c)->wint_t { return c; }); +END_SIMPLE_FUZZER() diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index e7e0ee247..baab07d31 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -48,6 +48,9 @@ namespace net_load_tests struct test_connection_context : epee::net_utils::connection_context_base { test_connection_context(): epee::net_utils::connection_context_base(boost::uuids::nil_uuid(), {}, false, false), m_closed(false) {} + static constexpr int handshake_command() noexcept { return 1001; } + static constexpr bool handshake_complete() noexcept { return true; } + size_t get_max_bytes(int command) const { return LEVIN_DEFAULT_MAX_PACKET_SIZE; } volatile bool m_closed; }; diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 33ef93288..556e0ec40 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -47,6 +47,7 @@ set(unit_tests_sources dns_resolver.cpp epee_boosted_tcp_server.cpp epee_levin_protocol_handler_async.cpp + epee_serialization.cpp epee_utils.cpp expect.cpp fee.cpp diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index ab6791324..a499fa608 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -43,6 +43,9 @@ namespace { struct test_levin_connection_context : public epee::net_utils::connection_context_base { + static constexpr int handshake_command() noexcept { return 1001; } + static constexpr bool handshake_complete() noexcept { return true; } + size_t get_max_bytes(int command) const { return LEVIN_DEFAULT_MAX_PACKET_SIZE; } }; typedef epee::levin::async_protocol_handler_config<test_levin_connection_context> test_levin_protocol_handler_config; @@ -193,6 +196,7 @@ namespace { m_handler_config.set_handler(m_pcommands_handler, [](epee::levin::levin_commands_handler<test_levin_connection_context> *handler) { delete handler; }); m_handler_config.m_invoke_timeout = invoke_timeout; + m_handler_config.m_initial_max_packet_size = max_packet_size; m_handler_config.m_max_packet_size = max_packet_size; } diff --git a/tests/unit_tests/epee_serialization.cpp b/tests/unit_tests/epee_serialization.cpp new file mode 100644 index 000000000..cade16f0d --- /dev/null +++ b/tests/unit_tests/epee_serialization.cpp @@ -0,0 +1,54 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <cstdint> +#include <gtest/gtest.h> + +#include "storages/portable_storage.h" + +TEST(epee_binary, two_keys) +{ + static constexpr const std::uint8_t data[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x08, 0x01, 'a', + 0x0B, 0x00, 0x01, 'b', 0x0B, 0x00 + }; + + epee::serialization::portable_storage storage{}; + EXPECT_TRUE(storage.load_from_binary(data)); +} + +TEST(epee_binary, duplicate_key) +{ + static constexpr const std::uint8_t data[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x08, 0x01, 'a', + 0x0B, 0x00, 0x01, 'a', 0x0B, 0x00 + }; + + epee::serialization::portable_storage storage{}; + EXPECT_FALSE(storage.load_from_binary(data)); +} diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index 2ad149afe..53a7f7b67 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -178,17 +178,17 @@ namespace { using base_type = epee::net_utils::connection_context_base; static_cast<base_type&>(context_) = base_type{random_generator(), {}, is_incoming, false}; + context_.m_state = cryptonote::cryptonote_connection_context::state_normal; handler_.after_init_connection(); } //\return Number of messages processed - std::size_t process_send_queue() + std::size_t process_send_queue(const bool valid = true) { std::size_t count = 0; for ( ; !endpoint_.send_queue_.empty(); ++count, endpoint_.send_queue_.pop_front()) { - // invalid messages shoudn't be possible in this test; - EXPECT_TRUE(handler_.handle_recv(endpoint_.send_queue_.front().data(), endpoint_.send_queue_.front().size())); + EXPECT_EQ(valid, handler_.handle_recv(endpoint_.send_queue_.front().data(), endpoint_.send_queue_.front().size())); } return count; } @@ -238,6 +238,13 @@ namespace return {connection, std::move(request)}; } + static received_message get_raw_message(std::deque<received_message>& queue) + { + received_message out{std::move(queue.front())}; + queue.pop_front(); + return out; + } + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, cryptonote::levin::detail::p2p_context& context) override final { buff_out = nullptr; @@ -294,6 +301,11 @@ namespace { return get_message<T>(notified_); } + + received_message get_raw_notification() + { + return get_raw_message(notified_); + } }; class levin_notify : public ::testing::Test @@ -322,6 +334,8 @@ namespace EXPECT_EQ(0u, events_.relayed_method_size()); } + cryptonote::levin::connections& get_connections() noexcept { return *connections_; } + void add_connection(const bool is_incoming) { contexts_.emplace_back(io_service_, *connections_, random_generator_, is_incoming); @@ -2140,3 +2154,27 @@ TEST_F(levin_notify, noise_stem) } } } + +TEST_F(levin_notify, command_max_bytes) +{ + static constexpr int ping_command = nodetool::COMMAND_PING::ID; + + add_connection(true); + + std::string bytes(4096, 'h'); + + EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1u, contexts_.front().process_send_queue(true)); + EXPECT_EQ(1u, receiver_.notified_size()); + + const received_message msg = receiver_.get_raw_notification(); + EXPECT_EQ(ping_command, msg.command); + EXPECT_EQ(contexts_.front().get_id(), msg.connection); + EXPECT_EQ(bytes, msg.payload); + + bytes.push_back('e'); + + EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1u, contexts_.front().process_send_queue(false)); + EXPECT_EQ(0u, receiver_.notified_size()); +} diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index 0569d3748..4d6f09e69 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -55,7 +55,8 @@ public: bool init(const boost::program_options::variables_map& vm) {return true ;} bool deinit(){return true;} bool get_short_chain_history(std::list<crypto::hash>& ids) const { return true; } - bool have_block(const crypto::hash& id) const {return true;} + bool have_block(const crypto::hash& id, int *where = NULL) const {return false;} + bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const {return false;} void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;} bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, cryptonote::relay_method tx_relay, bool relayed) { return true; } bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, cryptonote::relay_method tx_relay, bool relayed) { return true; } diff --git a/utils/health/README.md b/utils/health/README.md index dea46280e..8fadd3908 100644 --- a/utils/health/README.md +++ b/utils/health/README.md @@ -15,6 +15,10 @@ On the first run, the script will complain about the missing ClangBuildAnalyzer `utils/health/clang-tidy-run.sh` Performs Lint checks on the source code and stores the result in the build directory. More information on the [home page](https://clang.llvm.org/extra/clang-tidy/). +##include-what-you-use +`utils/health/clang-include-what-you-use-run.sh` +Analyses the header file hierarchy and delivers hints on how to reduce their complexity. More information on the [home page](https://include-what-you-use.org/). + ##Valgrind checks `utils/health/valgrind-tests.sh` diff --git a/utils/health/clang-include-what-you-use-run.sh b/utils/health/clang-include-what-you-use-run.sh new file mode 100755 index 000000000..655a188bd --- /dev/null +++ b/utils/health/clang-include-what-you-use-run.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Include What You Use analyses the complexity of your header hierarchy and proposes optimisations. +# User documentation: +# https://github.com/include-what-you-use/include-what-you-use/blob/master/README.md + +# Build variables +PROG="include-what-you-use" +PROG_SHORT="iwyu" +DIR_BUILD="build/clang-$PROG_SHORT" + +RESULT="$PROG_SHORT-result.txt" + +if hash "$PROG"; then + echo "Found: $PROG" +else + echo "Couldn't find: $PROG" + echo "Please run the below command to install $PROG:" + echo "sudo apt install $PROG_SHORT" + exit 1 +fi + +mkdir -p "$DIR_BUILD" && cd "$DIR_BUILD" +rm `find . -name "CMakeCache.txt"` || true + +UWYU_COMMAND="$PROG;-Xiwyu;any;-Xiwyu;iwyu;-Xiwyu;args" # Copy-pasted from the user docs. + +cmake ../.. \ +-DCMAKE_C_COMPILER=clang \ +-DCMAKE_CXX_COMPILER=clang++ \ +-DUSE_CCACHE=ON \ +-DCMAKE_C_INCLUDE_WHAT_YOU_USE="$UWYU_COMMAND" \ +-DCMAKE_CXX_INCLUDE_WHAT_YOU_USE="$UWYU_COMMAND" \ +-DBUILD_SHARED_LIBS=ON \ +-DBUILD_TESTS=ON + +make clean # Clean up to generate the full report +time make -k 2>&1 | tee "$RESULT" # Run the scan. -k means: ignore errors +#time make -k easylogging 2>&1 | tee $RESULT # Quick testing: build a single target +KPI=$(cat "$RESULT" | wc -l) +tar -cJvf "$RESULT.txz" "$RESULT" # Zip the result, because it's huge. +rm -v "$RESULT" + +echo "" +echo "Readable result stored in: $DIR_BUILD/$RESULT.gz" + +echo "$KPI" > "kpis.txt" diff --git a/utils/health/valgrind-tests.sh b/utils/health/valgrind-tests.sh new file mode 100755 index 000000000..9f5e7e7c0 --- /dev/null +++ b/utils/health/valgrind-tests.sh @@ -0,0 +1,161 @@ +#!/bin/bash -e + +# Copyright (c) 2014-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This script is able to run valgrind's callgrind, cachegrind and memcheck for a given set of executables. +# It expects ONE PARAMETER, which points to a file with paths to executables and their arguments, written line by line. + +if [ "$#" -ne 1 ]; then + echo "Please provide an argument, which points to a file with paths to executables and their arguments, written line by line. For example:" + echo "" + echo "ls -l -h" + echo "build/tests/unit_tests/unit_tests" + exit 1 +fi + +FILE_IN="$1" +DIR_OUT="build/valgrind-output" # Using build as the base output directory, as it's ignored in .gitignore + +function is_file_or_exit { + FILE="${1}" + if [ -f $FILE ]; then + echo "The input file $FILE exists. Can proceed." + else + echo "The input file $FILE doesn't exist." + exit 1 + fi + return 0 +} + +function is_tool_or_exit { + TOOL="${1}" + if $(hash ${TOOL}); then + echo "${TOOL} is installed. Can proceed." + else + echo "Please install ${TOOL} to continue." + exit 1 + fi + return 0 +} + +function get_tool_out_file_base { + EXE="${1}" + TOOL="${2}" + + EXE_NAME=$(basename $EXE) + local retval="${DIR_OUT}/${EXE_NAME}-${TOOL}" + echo "$retval" +} + +function get_tool_out_file { + EXE="${1}" + TOOL="${2}" + + FILE_OUT_BASE=$(get_tool_out_file_base ${EXE} ${TOOL}) + local retval="--${TOOL}-out-file=${FILE_OUT_BASE}.out" + echo "$retval" +} + +function run_valgrind_4_executable { + EXE="${1}" + ARGS="${2}" + TOOL="${3}" + EXTRA_OPTS="${4}" + FILE_OUT_TOOL="${5}" + FILE_OUT_BASE=$(get_tool_out_file_base ${EXE} ${TOOL}) + + echo "Runnig '${TOOL}' for '${EXE}' with args '${ARGS}'" + echo "EXTRA_OPTS = ${EXTRA_OPTS}" + echo "FILE_OUT_TOOL = ${FILE_OUT_TOOL}" + if ! valgrind --tool=${TOOL} ${FILE_OUT_TOOL} --log-file="${FILE_OUT_BASE}.log" ${EXTRA_OPTS} ${EXE} ${ARGS}; then + echo "FAILED in runnig ${TOOL} for ${EXE} !" + fi +} + +function run_valgrind_4_executable_callgrind { + EXE="${1}" + ARGS="${2}" + TOOL="callgrind" + EXTRA_OPTS="--dump-instr=yes --simulate-cache=yes --collect-jumps=yes" + FILE_OUT_TOOL=$(get_tool_out_file ${EXE} ${TOOL}) + + run_valgrind_4_executable ${EXE} "${ARGS}" ${TOOL} "${EXTRA_OPTS}" ${FILE_OUT_TOOL} +} + +function run_valgrind_4_executable_cachegrind { + EXE="${1}" + ARGS="${2}" + TOOL="cachegrind" + EXTRA_OPTS="" + FILE_OUT_TOOL=$(get_tool_out_file ${EXE} ${TOOL}) + + run_valgrind_4_executable ${EXE} "${ARGS}" ${TOOL} "${EXTRA_OPTS}" ${FILE_OUT_TOOL} +} + +function run_valgrind_4_executable_memcheck { + EXE="${1}" + ARGS="${2}" + TOOL="memcheck" + #EXTRA_OPTS="--leak-check=yes" # Minimalistic + EXTRA_OPTS="--leak-check=full --show-leak-kinds=all --track-origins=yes" + FILE_OUT_TOOL="" # memcheck has no special out file, only the log + + run_valgrind_4_executable ${EXE} "${ARGS}" ${TOOL} "${EXTRA_OPTS}" ${FILE_OUT_TOOL} +} + +function run_valgrind_4_executable_all { + EXE_ARGS_ARR=(${1}) + EXE=${EXE_ARGS_ARR[0]} # First element of the array + ARGS=${EXE_ARGS_ARR[@]:1} # Every next element + + #EXE="ls" # A quick check of the happy path + #EXE="nothere" # A quick check of error handling - no such executable + #EXE=/bin/false # A quick check of error handling - executable returned != 0 + + run_valgrind_4_executable_memcheck ${EXE} "${ARGS}" + run_valgrind_4_executable_cachegrind ${EXE} "${ARGS}" + run_valgrind_4_executable_callgrind ${EXE} "${ARGS}" +} + +is_tool_or_exit valgrind +is_file_or_exit "$FILE_IN" +echo "All OK." +echo "Will perform checks for the following executables and their arguments:" +while IFS= read -r line; do + echo "$line" +done < "$FILE_IN" + +mkdir -p "$DIR_OUT" +while IFS= read -r line; do + echo "$line" + run_valgrind_4_executable_all "$line" +done < "$FILE_IN" + +echo "Done. All data saved in ${DIR_OUT}" + |