diff options
Diffstat (limited to 'contrib')
29 files changed, 1177 insertions, 848 deletions
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 511f0416e..046115bd3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -26,5 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +monero_enable_coverage() add_subdirectory(epee) diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 25f8c648b..84dc79266 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -24,211 +24,23 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include <fstream> -#include <boost/filesystem/path.hpp> -#include <boost/filesystem/operations.hpp> -#ifdef WIN32 -#include <windows.h> -#include "string_tools.h" -#endif - -// On Windows there is a problem with non-ASCII characters in path and file names -// as far as support by the standard components used is concerned: - -// The various file stream classes, e.g. std::ifstream and std::ofstream, are -// part of the GNU C++ Library / libstdc++. On the most basic level they use the -// fopen() call as defined / made accessible to programs compiled within MSYS2 -// by the stdio.h header file maintained by the MinGW project. - -// The critical point: The implementation of fopen() is part of MSVCRT, the -// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any -// Unicode support. - -// Monero code that would want to continue to use the normal file stream classes -// but WITH Unicode support could therefore not solve this problem on its own, -// but 2 different projects from 2 different maintaining groups would need changes -// in this particular direction - something probably difficult to achieve and -// with a long time to wait until all new versions / releases arrive. - -// Implemented solution approach: Circumvent the problem by stopping to use std -// file stream classes on Windows and directly use Unicode-capable WIN32 API -// calls. Most of the code doing so is concentrated in this header file here. +#include <string> +#include <ctime> namespace epee { namespace file_io_utils { - inline - bool is_file_exist(const std::string& path) - { - boost::filesystem::path p(path); - return boost::filesystem::exists(p); - } - - inline - bool save_string_to_file(const std::string& path_to_file, const std::string& str) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD bytes_written; - DWORD bytes_to_write = (DWORD)str.size(); - BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); - CloseHandle(file_handle); - if (bytes_written != bytes_to_write) - result = FALSE; - return result; -#else - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool get_file_time(const std::string& path_to_file, time_t& ft) - { - boost::system::error_code ec; - ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); - if(!ec) - return true; - else - return false; - } - - inline - bool set_file_time(const std::string& path_to_file, const time_t& ft) - { - boost::system::error_code ec; - boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); - if(!ec) - return true; - else - return false; - } - - - inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { - CloseHandle(file_handle); - return false; - } - target_str.resize(file_size); - DWORD bytes_read; - BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); - CloseHandle(file_handle); - if (bytes_read != file_size) - result = FALSE; - return result; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - - std::ifstream::pos_type file_size = fstream.tellg(); - - if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large - return false;//don't go crazy - size_t file_size_t = static_cast<size_t>(file_size); - - target_str.resize(file_size_t); - - fstream.seekg (0, std::ios::beg); - fstream.read((char*)target_str.data(), target_str.size()); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool append_string_to_file(const std::string& path_to_file, const std::string& str) - { - // No special Windows implementation because so far not used in Monero code - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } - } - - inline - bool get_file_size(const std::string& path_to_file, uint64_t &size) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER file_size; - BOOL result = GetFileSizeEx(file_handle, &file_size); - CloseHandle(file_handle); - if (result) { - size = file_size.QuadPart; - } - return size; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - size = fstream.tellg(); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - + bool is_file_exist(const std::string& path); + bool save_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_time(const std::string& path_to_file, time_t& ft); + bool set_file_time(const std::string& path_to_file, const time_t& ft); + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000); + bool append_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_size(const std::string& path_to_file, uint64_t &size); } } diff --git a/contrib/epee/include/memwipe.h b/contrib/epee/include/memwipe.h index c6e8f072c..c6f247913 100644 --- a/contrib/epee/include/memwipe.h +++ b/contrib/epee/include/memwipe.h @@ -32,6 +32,7 @@ #ifdef __cplusplus #include <array> +#include <cstddef> extern "C" { #endif diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 5fffde8d5..522cdf263 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -47,7 +47,7 @@ #endif #include <iostream> -#include <boost/lexical_cast.hpp> +#include <ctime> #pragma once namespace epee @@ -115,15 +115,7 @@ namespace misc_utils } - - inline std::string get_thread_string_id() - { -#if defined(_WIN32) - return boost::lexical_cast<std::string>(GetCurrentThreadId()); -#elif defined(__GNUC__) - return boost::lexical_cast<std::string>(pthread_self()); -#endif - } + std::string get_thread_string_id(); inline bool get_gmt_time(time_t t, struct tm &tm) { diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 61e2b30fe..89971bea2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -42,7 +42,7 @@ #include <boost/make_shared.hpp> #include <boost/thread.hpp> #include "warnings.h" -#include "string_tools.h" +#include "string_tools_lexical.h" #include "misc_language.h" #include "net/local_ip.h" #include "pragma_comp_defs.h" diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index bf6589c92..4af4da790 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -27,14 +27,13 @@ #pragma once -#include <boost/lexical_cast.hpp> -#include <boost/regex.hpp> +#include "memwipe.h" + #include <boost/utility/string_ref.hpp> + #include <string> #include <utility> - -#include "memwipe.h" -#include "string_tools.h" +#include <list> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" @@ -66,34 +65,9 @@ namespace net_utils typedef std::list<std::pair<std::string, std::string> > fields_list; - inline - std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) - { - fields_list::const_iterator it = fields.begin(); - for(; it != fields.end(); it++) - if(!string_tools::compare_no_case(param_name, it->first)) - break; - - if(it==fields.end()) - return std::string(); + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields); - return it->second; - } - - - inline - std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) - { - std::string buff = "([\\?|&])"; - buff += param_name + "=([^&]*)"; - boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) - { - return result[2]; - } - return std::string(); - } + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri); static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 3725ef079..29ef82fb1 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -30,7 +30,6 @@ #include <ctype.h> #include <boost/shared_ptr.hpp> #include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include <boost/optional/optional.hpp> #include <boost/utility/string_ref.hpp> //#include <mbstring.h> @@ -46,6 +45,7 @@ #endif #include "string_tools.h" +#include "string_tools_lexical.h" #include "reg_exp_definer.h" #include "abstract_http_client.h" #include "http_base.h" diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index fce6d4b7e..df59a6c44 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,6 +31,7 @@ #include <cstdint> +#include "byte_stream.h" #include "net_utils_base.h" #include "span.h" @@ -83,11 +84,12 @@ namespace levin #define LEVIN_PROTOCOL_VER_0 0 #define LEVIN_PROTOCOL_VER_1 1 + template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; @@ -125,12 +127,41 @@ namespace levin } } + //! Provides space for levin (p2p) header, so that payload can be sent without copy + class message_writer + { + byte_slice finalize(uint32_t command, uint32_t flags, uint32_t return_code, bool expect_response); + public: + using header = bucket_head2; + + explicit message_writer(std::size_t reserve = 8192); + + message_writer(const message_writer&) = delete; + message_writer(message_writer&&) = default; + ~message_writer() = default; + message_writer& operator=(const message_writer&) = delete; + message_writer& operator=(message_writer&&) = default; + + //! \return Size of payload (excludes header size). + std::size_t payload_size() const noexcept + { + return buffer.size() < sizeof(header) ? 0 : buffer.size() - sizeof(header); + } + + byte_slice finalize_invoke(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, true); } + byte_slice finalize_notify(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, false); } + byte_slice finalize_response(uint32_t command, uint32_t return_code) + { + return finalize(command, LEVIN_PACKET_RESPONSE, return_code, false); + } + + //! Has space for levin header until a finalize method is used + byte_stream buffer; + }; + //! \return Intialized levin header. bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; - //! \return A levin notification message. - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); - /*! Generate a dummy levin message. \param noise_bytes Total size of the returned `byte_slice`. @@ -140,12 +171,11 @@ namespace levin /*! Generate 1+ levin messages that are identical to the noise message size. - \param noise Each levin message will be identical to the size of this - message. The bytes from this message will be used for padding. + \param noise_size Each levin message will be identical to this value. \return `nullptr` if `noise.size()` is less than the levin header size. Otherwise, a levin notification message OR 2+ levin fragment messages. Each message is `noise.size()` in length. */ - byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); + byte_slice make_fragmented_notify(const std::size_t noise_size, int command, message_writer message); } } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index d062fa877..a6816cafc 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -51,6 +51,21 @@ #define MIN_BYTES_WANTED 512 #endif +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char* category) +{ + MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") + << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); +} + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "command-%u", command); + on_levin_traffic(context, initiator, sent, error, bytes, buf); +} + namespace epee { namespace levin @@ -88,11 +103,10 @@ public: uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id); template<class callback_t> - int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); @@ -122,12 +136,17 @@ class async_protocol_handler { std::string m_fragment_buffer; - bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + bool send_message(byte_slice message) { - const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); - if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + if (message.size() < sizeof(message_writer::header)) + return false; + + message_writer::header head; + std::memcpy(std::addressof(head), message.data(), sizeof(head)); + if(!m_pservice_endpoint->do_send(std::move(message))) return false; + on_levin_traffic(m_connection_context, true, true, false, head.m_cb, head.m_command); MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb << ", flags" << head.m_flags << ", r?=" << head.m_have_to_return_data @@ -523,26 +542,17 @@ public: { if(m_current_head.m_have_to_return_data) { - byte_slice return_buff; + levin::message_writer return_message{32 * 1024}; const uint32_t return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + m_current_head.m_command, buff_to_invoke, return_message.buffer, m_connection_context ); // peer_id remains unset if dropped if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) m_max_packet_size = m_config.m_max_packet_size; - bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); - head.m_return_code = SWAP32LE(return_code); - - if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) + if(!send_message(return_message.finalize_response(m_current_head.m_command, return_code))) return false; - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", flags" << head.m_flags - << ", r?=" << head.m_have_to_return_data - <<", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); @@ -619,7 +629,7 @@ public: } template<class callback_t> - bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, message_writer in_msg, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -638,7 +648,7 @@ public: if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if(!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -664,7 +674,7 @@ public: return true; } - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out) + int invoke(int command, message_writer in_msg, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -676,7 +686,7 @@ public: if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if (!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; @@ -713,25 +723,9 @@ public: return m_invoke_result_code; } - int notify(int command, const epee::span<const uint8_t> in_buff) - { - misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( - boost::bind(&async_protocol_handler::finish_outer_call, this)); - - CRITICAL_REGION_LOCAL(m_call_lock); - - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) - { - LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); - return -1; - } - - return 1; - } - - /*! Sends `message` without adding a levin header. The message must have - been created with `make_notify`, `make_noise_notify` or - `make_fragmented_notify`. See additional instructions for + /*! Sends `message` without adding a levin header. The message must have been + created with `make_noise_notify`, `make_fragmented_notify`, or + `message_writer::finalize_notify`. See additional instructions for `make_fragmented_notify`. \return 1 on success */ @@ -741,14 +735,11 @@ public: boost::bind(&async_protocol_handler::finish_outer_call, this) ); - const std::size_t length = message.size(); - if (!m_pservice_endpoint->do_send(std::move(message))) + if (!send_message(std::move(message))) { LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -838,19 +829,19 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config<t_connection_context>::invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->invoke(command, in_buff, buff_out) : r; + return LEVIN_OK == r ? aph->invoke(command, std::move(in_msg), buff_out) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> -int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config<t_connection_context>::invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->async_invoke(command, in_buff, cb, timeout) : r; + return LEVIN_OK == r ? aph->async_invoke(command, std::move(in_msg), cb, timeout) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> @@ -929,14 +920,6 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id) -{ - async_protocol_handler<t_connection_context>* aph; - int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->notify(command, in_buff) : r; -} -//------------------------------------------------------------------------------------------ -template<class t_connection_context> int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) { async_protocol_handler<t_connection_context>* aph; diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index cf637ba1d..d794dd51c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -24,12 +24,9 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - - #pragma once + #include "http_base.h" -#include "reg_exp_definer.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -38,173 +35,8 @@ namespace epee { namespace net_utils { - - inline bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) - { - enum state - { - st_param_name, - st_param_val - }; - state st = st_param_name; - std::string::const_iterator start_it = query.begin(); - std::pair<std::string, std::string> e; - for(std::string::const_iterator it = query.begin(); it != query.end(); it++) - { - switch(st) - { - case st_param_name: - if(*it == '=') - { - e.first.assign(start_it, it); - start_it = it;++start_it; - st = st_param_val; - } - break; - case st_param_val: - if(*it == '&') - { - e.second.assign(start_it, it); - start_it = it;++start_it; - params.push_back(e); - e.first.clear();e.second.clear(); - st = st_param_name; - } - break; - default: - LOG_ERROR("Unknown state " << (int)st); - return false; - } - } - if(st == st_param_name) - { - if(start_it != query.end()) - { - e.first.assign(start_it, query.end()); - params.push_back(e); - } - }else - { - if(start_it != query.end()) - e.second.assign(start_it, query.end()); - - if(e.first.size()) - params.push_back(e); - } - return true; - } - - inline - bool parse_uri(const std::string uri, http::uri_content& content) - { - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - content.m_query_params.clear(); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - - boost::smatch result; - if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); - content.m_path = uri; - return true; - } - if(result[1].matched) - { - content.m_path = result[1]; - } - if(result[3].matched) - { - content.m_query = result[3]; - } - if(result[5].matched) - { - content.m_fragment = result[5]; - } - if(content.m_query.size()) - { - parse_uri_query(content.m_query, content.m_query_params); - } - return true; - } - - inline - bool parse_url_ipv6(const std::string url_str, http::url_content& content) - { - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 3 4 5 6 7 - - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return false; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - else // if host not matched, matching should be considered failed - { - return false; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - - inline - bool parse_url(const std::string url_str, http::url_content& content) - { - - if (parse_url_ipv6(url_str, content)) return true; - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 34 5 6 7 - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return true; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - + bool parse_uri(const std::string uri, http::uri_content& content); + bool parse_url_ipv6(const std::string url_str, http::url_content& content); + bool parse_url(const std::string url_str, http::url_content& content); } } diff --git a/contrib/epee/include/reg_exp_definer.h b/contrib/epee/include/reg_exp_definer.h index eb11c9e10..386a45f9a 100644 --- a/contrib/epee/include/reg_exp_definer.h +++ b/contrib/epee/include/reg_exp_definer.h @@ -29,6 +29,7 @@ #define _REG_EXP_DEFINER_H_ #include <boost/interprocess/detail/atomic.hpp> +#include <boost/regex.hpp> #include "syncobj.h" namespace epee diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 802e16c1b..383d67cc2 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -37,21 +37,14 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category); + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command); + namespace { - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category) - { - MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") - << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); - } - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) - { - char buf[32]; - snprintf(buf, sizeof(buf), "command-%u", command); - return on_levin_traffic(context, initiator, sent, error, bytes, buf); - } static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { 8192, // objects 16384, // fields @@ -117,12 +110,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; + levin::message_writer to_send{16 * 1024}; std::string buff_to_recv; - stg.store_to_binary(buff_to_send, 16 * 1024); + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); + int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); @@ -145,10 +137,9 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send, 16 * 1024); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + levin::message_writer to_send{16 * 1024}; + stg.store_to_binary(to_send.buffer); + int res = transport.invoke_async(command, std::move(to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -192,11 +183,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send); + levin::message_writer to_send; + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); + int res = transport.send(to_send.finalize_notify(command), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -207,7 +197,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff, &default_levin_limits)) @@ -230,12 +220,11 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out, 32 * 1024)) + if(!strg_out.store_to_binary(buff_out)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; } - on_levin_traffic(context, false, true, false, buff_out.size(), command); return res; } @@ -262,7 +251,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -271,13 +260,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; epee::byte_slice fake_str; \ - return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ + bool handled = false; epee::byte_stream fake_str; \ + return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -297,7 +286,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index c5d0c48ee..655a2eb12 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -34,6 +34,7 @@ namespace epee { class byte_slice; + class byte_stream; namespace serialization { /************************************************************************/ @@ -83,8 +84,13 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); - bool load_from_binary(const std::string& target, const limits_t *limits = NULL); + bool store_to_binary(byte_stream& ss); + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = nullptr); + bool load_from_binary(const std::string& target, const limits_t *limits = nullptr) + { + return load_from_binary(epee::strspan<uint8_t>(target), limits); + } + template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index e7250e895..7f6596f36 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -29,13 +29,15 @@ #include <string> #include "byte_slice.h" -#include "parserse_base_utils.h" +#include "parserse_base_utils.h" /// TODO: (mj-xmr) This will be reduced in an another PR #include "portable_storage.h" #include "file_io_utils.h" #include "span.h" namespace epee { + class byte_stream; + namespace serialization { //----------------------------------------------------------------------------------------------------------- @@ -127,5 +129,14 @@ namespace epee store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } + //----------------------------------------------------------------------------------------------------------- + template<class t_struct> + bool store_t_to_binary(t_struct& str_in, byte_stream& binary_buff) + { + portable_storage ps; + str_in.store(ps); + return ps.store_to_binary(binary_buff); + } + } } diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 2d9317d60..dbbe1906e 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -24,33 +24,16 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #ifndef _STRING_TOOLS_H_ #define _STRING_TOOLS_H_ -// Previously pulled in by ASIO, further cleanup still required ... -#ifdef _WIN32 -# include <winsock2.h> -# include <windows.h> -#endif - -#include <string.h> -#include <locale> -#include <cstdlib> -#include <string> -#include <type_traits> -#include <boost/lexical_cast.hpp> -#include <boost/algorithm/string/predicate.hpp> -#include <boost/utility/string_ref.hpp> -#include "misc_log_ex.h" -#include "storages/parserse_base_utils.h" #include "hex.h" -#include "memwipe.h" #include "mlocker.h" -#include "span.h" -#include "warnings.h" +#include <boost/utility/string_ref.hpp> +#include <sstream> +#include <string> +#include <cstdint> #ifndef OUT #define OUT @@ -74,202 +57,38 @@ namespace string_tools { return from_hex::to_string(res, s); } - //---------------------------------------------------------------------------- -PUSH_WARNINGS -DISABLE_GCC_WARNING(maybe-uninitialized) - template<class XType> - inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) - { - if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) - { - for (char c : str_id) - { - if (!epee::misc_utils::parse::isdigit(c)) - return false; - } - } - - try - { - val = boost::lexical_cast<XType>(str_id); - return true; - } - catch(const std::exception& /*e*/) - { - //const char* pmsg = e.what(); - return false; - } - catch(...) - { - return false; - } - - return true; - } -POP_WARNINGS - //---------------------------------------------------------------------------- - template<class XType> - inline bool xtype_to_string(const XType& val, std::string& str) - { - try - { - str = boost::lexical_cast<std::string>(val); - } - catch(...) - { - return false; - } - - return true; - } - //---------------------------------------------------------------------------- - std::string get_ip_string_from_int32(uint32_t ip); - //---------------------------------------------------------------------------- - bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); - //---------------------------------------------------------------------------- - inline bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) - { - //parse ip and address - std::string::size_type p = addres.find(':'); - std::string ip_str, port_str; - if(p == std::string::npos) - { - port = 0; - ip_str = addres; - } - else - { - ip_str = addres.substr(0, p); - port_str = addres.substr(p+1, addres.size()); - } - - if(!get_ip_int32_from_string(ip, ip_str)) - { - return false; - } - - if(p != std::string::npos && !get_xtype_from_string(port, port_str)) - { - return false; - } - return true; - } - - inline std::string num_to_string_fast(int64_t val) - { - /* - char buff[30] = {0}; - i64toa_s(val, buff, sizeof(buff)-1, 10); - return buff;*/ - return boost::lexical_cast<std::string>(val); - } - //---------------------------------------------------------------------------- - template<typename T> - inline std::string to_string_hex(const T &val) - { - static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); - std::stringstream ss; - ss << std::hex << val; - std::string s; - ss >> s; - return s; - } - //---------------------------------------------------------------------------- + + std::string get_ip_string_from_int32(uint32_t ip); + bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres); + std::string num_to_string_fast(int64_t val); - inline bool compare_no_case(const std::string& str1, const std::string& str2) - { - - return !boost::iequals(str1, str2); - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_name() - { - static std::string module_name; - return module_name; - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_folder() - { - static std::string module_folder; - return module_folder; - } - //---------------------------------------------------------------------------- + bool compare_no_case(const std::string& str1, const std::string& str2); + std::string& get_current_module_name(); + std::string& get_current_module_folder(); #ifdef _WIN32 - inline std::string get_current_module_path() - { - char pname [5000] = {0}; - GetModuleFileNameA( NULL, pname, sizeof(pname)); - pname[sizeof(pname)-1] = 0; //be happy ;) - return pname; - } + std::string get_current_module_path(); #endif - //---------------------------------------------------------------------------- - inline bool set_module_name_and_folder(const std::string& path_to_process_) - { - std::string path_to_process = path_to_process_; -#ifdef _WIN32 - path_to_process = get_current_module_path(); -#endif - std::string::size_type a = path_to_process.rfind( '\\' ); - if(a == std::string::npos ) - { - a = path_to_process.rfind( '/' ); - } - if ( a != std::string::npos ) - { - get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); - get_current_module_folder() = path_to_process.substr(0, a); - return true; - }else - return false; - - } - - //---------------------------------------------------------------------------- - inline bool trim_left(std::string& str) - { - for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) - str.erase(str.begin()); - - return true; - } - //---------------------------------------------------------------------------- - inline bool trim_right(std::string& str) - { - - for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) - str.erase( --((it++).base())); - - return true; - } - //---------------------------------------------------------------------------- - inline std::string& trim(std::string& str) - { - - trim_left(str); - trim_right(str); - return str; - } + bool set_module_name_and_folder(const std::string& path_to_process_); + bool trim_left(std::string& str); + bool trim_right(std::string& str); //---------------------------------------------------------------------------- - inline std::string trim(const std::string& str_) + inline std::string& trim(std::string& str) { - std::string str = str_; trim_left(str); trim_right(str); return str; } //---------------------------------------------------------------------------- - inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false) + inline std::string trim(const std::string& str_) { - if (s.size() < n) - { - if (prepend) - s = std::string(n - s.size(), c) + s; - else - s.append(n - s.size(), c); - } - return s; + std::string str = str_; + trim_left(str); + trim_right(str); + return str; } + std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false); + //---------------------------------------------------------------------------- template<class t_pod_type> std::string pod_to_hex(const t_pod_type& s) @@ -296,64 +115,25 @@ POP_WARNINGS { return hex_to_pod(hex_str, unwrap(s)); } - //---------------------------------------------------------------------------- - bool validate_hex(uint64_t length, const std::string& str); - //---------------------------------------------------------------------------- - inline std::string get_extension(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return res; - - res = str.substr(pos+1, str.size()-pos); - return res; - } - //---------------------------------------------------------------------------- - inline std::string cut_off_extension(const std::string& str) + //---------------------------------------------------------------------------- + template<typename T> + inline std::string to_string_hex(const T &val) { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return str; - - res = str.substr(0, pos); - return res; + static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); + std::stringstream ss; + ss << std::hex << val; + std::string s; + ss >> s; + return s; } - //---------------------------------------------------------------------------- + + bool validate_hex(uint64_t length, const std::string& str); + std::string get_extension(const std::string& str); + std::string cut_off_extension(const std::string& str); + #ifdef _WIN32 - inline std::wstring utf8_to_utf16(const std::string& str) - { - if (str.empty()) - return {}; - int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); - if (wstr_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::wstring wstr(wstr_size, wchar_t{}); - if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return wstr; - } - inline std::string utf16_to_utf8(const std::wstring& wstr) - { - if (wstr.empty()) - return {}; - int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); - if (str_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::string str(str_size, char{}); - if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return str; - } + std::wstring utf8_to_utf16(const std::string& str); + std::string utf16_to_utf8(const std::wstring& wstr); #endif } } diff --git a/contrib/epee/include/string_tools_lexical.h b/contrib/epee/include/string_tools_lexical.h new file mode 100644 index 000000000..01be562bc --- /dev/null +++ b/contrib/epee/include/string_tools_lexical.h @@ -0,0 +1,91 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef _STRING_TOOLS_LEXICAL_H_ +#define _STRING_TOOLS_LEXICAL_H_ + +#include "warnings.h" +#include "storages/parserse_base_utils.h" +#include <boost/lexical_cast.hpp> // A heavy header, that was extracted from the rest + +#ifndef OUT + #define OUT +#endif + +namespace epee +{ +namespace string_tools +{ +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) + template<class XType> + inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) + { + if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) + { + for (char c : str_id) + { + if (!epee::misc_utils::parse::isdigit(c)) + return false; + } + } + + try + { + val = boost::lexical_cast<XType>(str_id); + return true; + } + catch(const std::exception& /*e*/) + { + //const char* pmsg = e.what(); + return false; + } + catch(...) + { + return false; + } + + return true; + } +POP_WARNINGS + + template<class XType> + inline bool xtype_to_string(const XType& val, std::string& str) + { + try + { + str = boost::lexical_cast<std::string>(val); + } + catch(...) + { + return false; + } + + return true; + } +} +} +#endif //_STRING_TOOLS_LEXICAL_H_ diff --git a/contrib/epee/include/tiny_ini.h b/contrib/epee/include/tiny_ini.h index 2bc71fc1a..6ced548eb 100644 --- a/contrib/epee/include/tiny_ini.h +++ b/contrib/epee/include/tiny_ini.h @@ -28,8 +28,6 @@ #ifndef _TINY_INI_H_ #define _TINY_INI_H_ -#include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include "string_tools.h" namespace epee @@ -37,20 +35,8 @@ namespace epee namespace tiny_ini { - inline - bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) - { - std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; - const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) - return false; - res = result[2]; - string_tools::trim(res); - return true; - } - inline - std::string get_param_value(const std::string& param_name, const std::string& ini_entry) + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res); + inline std::string get_param_value(const std::string& param_name, const std::string& ini_entry) { std::string buff; get_param_value(param_name, ini_entry, buff); diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index f89d6f8cc..0f0a6ecad 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -35,7 +35,11 @@ add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_cli wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp int-util.cpp portable_storage.cpp misc_language.cpp - + misc_os_dependent.cpp + file_io_utils.cpp + net_parse_helpers.cpp + http_base.cpp + tiny_ini.cpp ${EPEE_HEADERS_PUBLIC} ) diff --git a/contrib/epee/src/abstract_http_client.cpp b/contrib/epee/src/abstract_http_client.cpp index 540917873..3ae09c90e 100644 --- a/contrib/epee/src/abstract_http_client.cpp +++ b/contrib/epee/src/abstract_http_client.cpp @@ -1,6 +1,7 @@ #include "net/abstract_http_client.h" #include "net/http_base.h" #include "net/net_parse_helpers.h" +#include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" diff --git a/contrib/epee/src/file_io_utils.cpp b/contrib/epee/src/file_io_utils.cpp new file mode 100644 index 000000000..5072adcbc --- /dev/null +++ b/contrib/epee/src/file_io_utils.cpp @@ -0,0 +1,231 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "file_io_utils.h" + +#include <fstream> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> +#ifdef WIN32 +#include <windows.h> +#include "string_tools.h" +#endif + +// On Windows there is a problem with non-ASCII characters in path and file names +// as far as support by the standard components used is concerned: + +// The various file stream classes, e.g. std::ifstream and std::ofstream, are +// part of the GNU C++ Library / libstdc++. On the most basic level they use the +// fopen() call as defined / made accessible to programs compiled within MSYS2 +// by the stdio.h header file maintained by the MinGW project. + +// The critical point: The implementation of fopen() is part of MSVCRT, the +// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any +// Unicode support. + +// Monero code that would want to continue to use the normal file stream classes +// but WITH Unicode support could therefore not solve this problem on its own, +// but 2 different projects from 2 different maintaining groups would need changes +// in this particular direction - something probably difficult to achieve and +// with a long time to wait until all new versions / releases arrive. + +// Implemented solution approach: Circumvent the problem by stopping to use std +// file stream classes on Windows and directly use Unicode-capable WIN32 API +// calls. Most of the code doing so is concentrated in this header file here. + +namespace epee +{ +namespace file_io_utils +{ + + bool is_file_exist(const std::string& path) + { + boost::filesystem::path p(path); + return boost::filesystem::exists(p); + } + + + bool save_string_to_file(const std::string& path_to_file, const std::string& str) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD bytes_written; + DWORD bytes_to_write = (DWORD)str.size(); + BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); + CloseHandle(file_handle); + if (bytes_written != bytes_to_write) + result = FALSE; + return result; +#else + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool get_file_time(const std::string& path_to_file, time_t& ft) + { + boost::system::error_code ec; + ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); + if(!ec) + return true; + else + return false; + } + + + bool set_file_time(const std::string& path_to_file, const time_t& ft) + { + boost::system::error_code ec; + boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); + if(!ec) + return true; + else + return false; + } + + + + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD file_size = GetFileSize(file_handle, NULL); + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { + CloseHandle(file_handle); + return false; + } + target_str.resize(file_size); + DWORD bytes_read; + BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); + CloseHandle(file_handle); + if (bytes_read != file_size) + result = FALSE; + return result; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + + std::ifstream::pos_type file_size = fstream.tellg(); + + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large + return false;//don't go crazy + size_t file_size_t = static_cast<size_t>(file_size); + + target_str.resize(file_size_t); + + fstream.seekg (0, std::ios::beg); + fstream.read((char*)target_str.data(), target_str.size()); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool append_string_to_file(const std::string& path_to_file, const std::string& str) + { + // No special Windows implementation because so far not used in Monero code + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } + } + + + bool get_file_size(const std::string& path_to_file, uint64_t &size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER file_size; + BOOL result = GetFileSizeEx(file_handle, &file_size); + CloseHandle(file_handle); + if (result) { + size = file_size.QuadPart; + } + return size; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + size = fstream.tellg(); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + +} +} diff --git a/contrib/epee/src/http_base.cpp b/contrib/epee/src/http_base.cpp new file mode 100644 index 000000000..647dfb899 --- /dev/null +++ b/contrib/epee/src/http_base.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/http_base.h" +#include "memwipe.h" +#include "string_tools.h" + +#include <boost/regex.hpp> +#include <string> +#include <utility> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.http" + +namespace epee +{ +namespace net_utils +{ +namespace http +{ + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) + { + fields_list::const_iterator it = fields.begin(); + for(; it != fields.end(); it++) + if(!string_tools::compare_no_case(param_name, it->first)) + break; + + if(it==fields.end()) + return std::string(); + + return it->second; + } + + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) + { + std::string buff = "([\\?|&])"; + buff += param_name + "=([^&]*)"; + boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) + { + return result[2]; + } + return std::string(); + } +} +} +} diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp index 5ec86b3d6..7c5cd5a78 100644 --- a/contrib/epee/src/levin_base.cpp +++ b/contrib/epee/src/levin_base.cpp @@ -34,6 +34,25 @@ namespace epee { namespace levin { + message_writer::message_writer(const std::size_t reserve) + : buffer() + { + buffer.reserve(reserve); + buffer.put_n(0, sizeof(header)); + } + + byte_slice message_writer::finalize(const uint32_t command, const uint32_t flags, const uint32_t return_code, const bool expect_response) + { + if (buffer.size() < sizeof(header)) + throw std::runtime_error{"levin_writer::finalize already called"}; + + header head = make_header(command, payload_size(), flags, expect_response); + head.m_return_code = SWAP32LE(return_code); + + std::memcpy(buffer.tellp() - buffer.size(), std::addressof(head), sizeof(head)); + return byte_slice{std::move(buffer)}; + } + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept { bucket_head2 head = {0}; @@ -47,12 +66,6 @@ namespace levin return head; } - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) - { - const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); - return byte_slice{epee::as_byte_span(head), payload}; - } - byte_slice make_noise_notify(const std::size_t noise_bytes) { static constexpr const std::uint32_t flags = @@ -68,46 +81,40 @@ namespace levin return byte_slice{std::move(buffer)}; } - byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + byte_slice make_fragmented_notify(const std::size_t noise_size, const int command, message_writer message) { - const size_t noise_size = noise_message.size(); if (noise_size < sizeof(bucket_head2) * 2) return nullptr; - if (payload.size() <= noise_size - sizeof(bucket_head2)) + if (message.buffer.size() <= noise_size) { /* The entire message can be sent at once, and the levin binary parser will ignore extra bytes. So just pad with zeroes and otherwise send a "normal", not fragmented message. */ - const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); - const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; - const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); - return byte_slice{as_byte_span(head), payload, padding_bytes}; + message.buffer.put_n(0, noise_size - message.buffer.size()); + return message.finalize_notify(command); } // fragment message + const byte_slice payload_bytes = message.finalize_notify(command); + span<const std::uint8_t> payload = to_span(payload_bytes); + const size_t payload_space = noise_size - sizeof(bucket_head2); const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; - std::string buffer{}; - buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + byte_stream buffer{}; + buffer.reserve(expected_fragments * noise_size); - bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + bucket_head2 head = make_header(0, payload_space, LEVIN_PACKET_BEGIN, false); + buffer.write(as_byte_span(head)); - head.m_command = command; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_cb = payload.size(); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + // internal levin header is in payload already - size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + size_t copy_size = payload.remove_prefix(payload_space); + buffer.write(payload.data() - copy_size, copy_size); - head.m_command = 0; head.m_flags = 0; - head.m_cb = noise_size - sizeof(bucket_head2); - while (!payload.empty()) { copy_size = payload.remove_prefix(payload_space); @@ -115,12 +122,12 @@ namespace levin if (payload.empty()) head.m_flags = LEVIN_PACKET_END; - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + buffer.write(as_byte_span(head)); + buffer.write(payload.data() - copy_size, copy_size); } const size_t padding = noise_size - copy_size - sizeof(bucket_head2); - buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + buffer.put_n(0, padding); return byte_slice{std::move(buffer)}; } diff --git a/contrib/epee/src/misc_os_dependent.cpp b/contrib/epee/src/misc_os_dependent.cpp new file mode 100644 index 000000000..cd4967131 --- /dev/null +++ b/contrib/epee/src/misc_os_dependent.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_os_dependent.h" +#include <boost/lexical_cast.hpp> + +namespace epee +{ +namespace misc_utils +{ + // TODO: (vtnerd) This function is weird since boost::this_thread::get_id() exists but returns a different value. + std::string get_thread_string_id() + { +#if defined(_WIN32) + return boost::lexical_cast<std::string>(GetCurrentThreadId()); +#elif defined(__GNUC__) + return boost::lexical_cast<std::string>(pthread_self()); +#endif + } +} +} diff --git a/contrib/epee/src/net_parse_helpers.cpp b/contrib/epee/src/net_parse_helpers.cpp new file mode 100644 index 000000000..de7843b67 --- /dev/null +++ b/contrib/epee/src/net_parse_helpers.cpp @@ -0,0 +1,206 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/net_parse_helpers.h" +#include "net/http_base.h" +#include "misc_log_ex.h" +#include "reg_exp_definer.h" +#include <boost/lexical_cast.hpp> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net" + +namespace epee +{ +namespace net_utils +{ + + static bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) + { + enum state + { + st_param_name, + st_param_val + }; + state st = st_param_name; + std::string::const_iterator start_it = query.begin(); + std::pair<std::string, std::string> e; + for(std::string::const_iterator it = query.begin(); it != query.end(); it++) + { + switch(st) + { + case st_param_name: + if(*it == '=') + { + e.first.assign(start_it, it); + start_it = it;++start_it; + st = st_param_val; + } + break; + case st_param_val: + if(*it == '&') + { + e.second.assign(start_it, it); + start_it = it;++start_it; + params.push_back(e); + e.first.clear();e.second.clear(); + st = st_param_name; + } + break; + default: + LOG_ERROR("Unknown state " << (int)st); + return false; + } + } + if(st == st_param_name) + { + if(start_it != query.end()) + { + e.first.assign(start_it, query.end()); + params.push_back(e); + } + }else + { + if(start_it != query.end()) + e.second.assign(start_it, query.end()); + + if(e.first.size()) + params.push_back(e); + } + return true; + } + + bool parse_uri(const std::string uri, http::uri_content& content) + { + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + content.m_query_params.clear(); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + + boost::smatch result; + if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); + content.m_path = uri; + return true; + } + if(result[1].matched) + { + content.m_path = result[1]; + } + if(result[3].matched) + { + content.m_query = result[3]; + } + if(result[5].matched) + { + content.m_fragment = result[5]; + } + if(content.m_query.size()) + { + parse_uri_query(content.m_query, content.m_query_params); + } + return true; + } + + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 + + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + bool parse_url(const std::string url_str, http::url_content& content) + { + + if (parse_url_ipv6(url_str, content)) return true; + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 34 5 6 7 + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return true; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + +} +} diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index c3c9ccc02..b922cc9e3 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -48,15 +48,23 @@ namespace serialization TRY_ENTRY(); byte_stream ss; ss.reserve(initial_buffer_size); + store_to_binary(ss); + target = epee::byte_slice{std::move(ss)}; + return true; + CATCH_ENTRY("portable_storage::store_to_binary", false); + } + + bool portable_storage::store_to_binary(byte_stream& ss) + { + TRY_ENTRY(); storage_block_header sbh{}; sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; ss.write(epee::as_byte_span(sbh)); pack_entry_to_buff(ss, m_root); - target = epee::byte_slice{std::move(ss)}; return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) + CATCH_ENTRY("portable_storage::store_to_binary", false); } bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) @@ -76,11 +84,6 @@ namespace serialization CATCH_ENTRY("portable_storage::load_from_json", false) } - bool portable_storage::load_from_binary(const std::string& target, const limits_t *limits) - { - return load_from_binary(epee::strspan<uint8_t>(target), limits); - } - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) { m_root.m_entries.clear(); diff --git a/contrib/epee/src/string_tools.cpp b/contrib/epee/src/string_tools.cpp index fd0254016..984a151b5 100644 --- a/contrib/epee/src/string_tools.cpp +++ b/contrib/epee/src/string_tools.cpp @@ -25,6 +25,29 @@ // #include "string_tools.h" +#include "string_tools_lexical.h" + + +// Previously pulled in by ASIO, further cleanup still required ... +#ifdef _WIN32 +# include <winsock2.h> +# include <windows.h> +#endif + +#include <locale> +#include <cstdlib> +#include <string> +#include <type_traits> +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/utility/string_ref.hpp> +#include "misc_log_ex.h" +#include "storages/parserse_base_utils.h" +#include "hex.h" +#include "memwipe.h" +#include "mlocker.h" +#include "span.h" +#include "warnings.h" #include <ctype.h> @@ -68,6 +91,180 @@ namespace string_tools return false; return true; } + //---------------------------------------------------------------------------- + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) + { + //parse ip and address + std::string::size_type p = addres.find(':'); + std::string ip_str, port_str; + if(p == std::string::npos) + { + port = 0; + ip_str = addres; + } + else + { + ip_str = addres.substr(0, p); + port_str = addres.substr(p+1, addres.size()); + } + + if(!get_ip_int32_from_string(ip, ip_str)) + { + return false; + } + + if(p != std::string::npos && !get_xtype_from_string(port, port_str)) + { + return false; + } + return true; + } + + std::string num_to_string_fast(int64_t val) + { + /* + char buff[30] = {0}; + i64toa_s(val, buff, sizeof(buff)-1, 10); + return buff;*/ + return boost::lexical_cast<std::string>(val); + } + + + bool compare_no_case(const std::string& str1, const std::string& str2) + { + + return !boost::iequals(str1, str2); + } + //---------------------------------------------------------------------------- + std::string& get_current_module_name() + { + static std::string module_name; + return module_name; + } + //---------------------------------------------------------------------------- + std::string& get_current_module_folder() + { + static std::string module_folder; + return module_folder; + } + +#ifdef _WIN32 + std::string get_current_module_path() + { + char pname [5000] = {0}; + GetModuleFileNameA( NULL, pname, sizeof(pname)); + pname[sizeof(pname)-1] = 0; //be happy ;) + return pname; + } +#endif + + bool set_module_name_and_folder(const std::string& path_to_process_) + { + std::string path_to_process = path_to_process_; +#ifdef _WIN32 + path_to_process = get_current_module_path(); +#endif + std::string::size_type a = path_to_process.rfind( '\\' ); + if(a == std::string::npos ) + { + a = path_to_process.rfind( '/' ); + } + if ( a != std::string::npos ) + { + get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); + get_current_module_folder() = path_to_process.substr(0, a); + return true; + }else + return false; + + } + + //---------------------------------------------------------------------------- + bool trim_left(std::string& str) + { + for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) + str.erase(str.begin()); + + return true; + } + //---------------------------------------------------------------------------- + bool trim_right(std::string& str) + { + + for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) + str.erase( --((it++).base())); + + return true; + } + //---------------------------------------------------------------------------- + std::string pad_string(std::string s, size_t n, char c, bool prepend) + { + if (s.size() < n) + { + if (prepend) + s = std::string(n - s.size(), c) + s; + else + s.append(n - s.size(), c); + } + return s; + } + + std::string get_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return res; + + res = str.substr(pos+1, str.size()-pos); + return res; + } + //---------------------------------------------------------------------------- + std::string cut_off_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return str; + + res = str.substr(0, pos); + return res; + } + //---------------------------------------------------------------------------- +#ifdef _WIN32 + std::wstring utf8_to_utf16(const std::string& str) + { + if (str.empty()) + return {}; + int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); + if (wstr_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::wstring wstr(wstr_size, wchar_t{}); + if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return wstr; + } + std::string utf16_to_utf8(const std::wstring& wstr) + { + if (wstr.empty()) + return {}; + int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); + if (str_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::string str(str_size, char{}); + if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return str; + } +#endif } } diff --git a/contrib/epee/src/tiny_ini.cpp b/contrib/epee/src/tiny_ini.cpp new file mode 100644 index 000000000..577ebf7c6 --- /dev/null +++ b/contrib/epee/src/tiny_ini.cpp @@ -0,0 +1,46 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "string_tools.h" +#include <boost/regex.hpp> + +namespace epee +{ +namespace tiny_ini +{ + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) + { + std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; + const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) + return false; + res = result[2]; + string_tools::trim(res); + return true; + } +} +} diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index e2e1d0b94..0a40d4608 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -36,6 +36,10 @@ This guide explains how to set up the environment, and how to start the builds. You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. +```bash +sudo adduser gitianuser +sudo usermod -aG sudo gitianuser +``` LXC --- @@ -83,7 +87,7 @@ Docker Prepare for building with docker: ```bash -sudo apt-get install git make curl docker.io +sudo bash -c 'apt-get update && apt-get upgrade -y && apt-get install git curl docker.io' ``` Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): @@ -96,13 +100,12 @@ sudo usermod -aG docker gitianuser Optionally add yourself to the docker group. Note that this will give docker root access to your system. ```bash -sudo usermod -aG docker gitianuser +sudo usermod -aG docker $USER ``` Manual Building ------------------- -The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. ======= The script automatically installs some packages with apt. If you are not running it on a debian-like system, pass `--no-apt` along with the other arguments to it. It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. @@ -122,17 +125,23 @@ cp monero/contrib/gitian/gitian-build.py . ### Setup the required environment -Setup for LXC: +Common setup part: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 +su - gitianuser -./gitian-build.py --setup $GH_USER $VERSION +GH_USER=YOUR_GITHUB_USER_NAME +VERSION=v0.17.2.0 ``` Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. +Setup for LXC: + +```bash +./gitian-build.py --setup $GH_USER $VERSION +``` + Setup for docker: ```bash @@ -145,8 +154,10 @@ fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) or pass the signed assert file back to your build machine. ```bash -git clone git@github.com:monero-project/gitian.sigs.git -git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git +git clone https://github.com/monero-project/gitian.sigs/ +pushd gitian.sigs +git remote add $GH_USER https://github.com/$GH_USER/gitian.sigs +popd ``` Build the binaries @@ -154,13 +165,26 @@ Build the binaries **Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. +Currently working MacOS solution: + +```bash +curl -O https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz +mv MacOSX10.11.sdk.tar.gz builder/inputs +``` + To build the most recent tag (pass in `--docker` if using docker): ```bash ./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION ``` -To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. +To speed up the build, use `-j 5 --memory 10000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 10000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. A good rule of thumb is, that Monero currently needs about 2 GB of RAM per core. + +A full example for `docker` would look like the following: + +```bash +./gitian-build.py -j 5 --memory 10000 --docker --detach-sign --no-commit --build $GH_USER $VERSION +``` If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. @@ -171,6 +195,22 @@ Take a look in the assert files and note the SHA256 checksums listed there. You should verify that the checksum that is listed matches each of the binaries you actually built. This may be done on Linux using the `sha256sum` command or on MacOS using `shasum --algorithm 256` for example. +An example script to verify the checksums would be: + +```bash +pushd out/${VERSION} + +for ASSERT in ../../sigs/${VERSION}-*/*/*.assert; do + if ! sha256sum --ignore-missing -c "${ASSERT}" ; then + echo "FAILED for ${ASSERT} ! Please inspect manually." + fi +done + +popd +``` + +Don't ignore the incorrect formatting of the found assert files. These files you'll have to compare manually (currently OSX and FreeBSD). + You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. @@ -181,14 +221,7 @@ Signing assert files If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 - -gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert -gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert -gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert -gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert -gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert +for ASSERT in sigs/${VERSION}-*/*/*.assert; do gpg --detach-sign ${ASSERT}; done ``` This will create a `.sig` file for each `.assert` file above (2 files for each platform). @@ -201,6 +234,7 @@ Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash +cd gitian.sigs git checkout -b $VERSION # add your assert and sig files... git commit -S -a -m "Add $GH_USER $VERSION" diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index a8140a8a6..6bf936958 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -94,10 +94,6 @@ def build(): os.chdir('builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) - subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) - subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', 'inputs/monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) rebuild() |