diff options
Diffstat (limited to 'contrib')
38 files changed, 1467 insertions, 424 deletions
diff --git a/contrib/depends/README.md b/contrib/depends/README.md index 6a15055b4..10866acbe 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -33,6 +33,7 @@ Common `host-platform-triplets` for cross compilation are: - `x86_64-apple-darwin11` for MacOSX - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit +- `riscv64-linux-gnu` for Linux RISCV 64 bit No other options are needed, the paths are automatically configured. diff --git a/contrib/depends/packages/icu4c.mk b/contrib/depends/packages/icu4c.mk index dc089bbbe..58ae637b0 100644 --- a/contrib/depends/packages/icu4c.mk +++ b/contrib/depends/packages/icu4c.mk @@ -1,8 +1,8 @@ package=icu4c -$(package)_version=55.1 -$(package)_download_path=https://github.com/TheCharlatan/icu4c/archive -$(package)_file_name=55.1.tar.gz -$(package)_sha256_hash=1f912c54035533fb4268809701d65c7468d00e292efbc31e6444908450cc46ef +$(package)_version=55.2 +$(package)_download_path=https://github.com/unicode-org/icu/releases/download/release-55-2/ +$(package)_file_name=$(package)-55_2-src.tgz +$(package)_sha256_hash=eda2aa9f9c787748a2e2d310590720ca8bcc6252adf6b4cfb03b65bef9d66759 $(package)_patches=icu-001-dont-build-static-dynamic-twice.patch define $(package)_set_vars diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index ea4902170..6fbcc3466 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -12,6 +12,10 @@ define $(package)_set_vars $(package)_config_opts_linux=--with-pic endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index e39dc1d04..8d08900f5 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -40,6 +40,7 @@ $(package)_config_opts_x86_64_linux=linux-x86_64 $(package)_config_opts_i686_linux=linux-generic32 $(package)_config_opts_arm_linux=linux-generic32 $(package)_config_opts_aarch64_linux=linux-generic64 +$(package)_config_opts_riscv64_linux=linux-generic64 $(package)_config_opts_mipsel_linux=linux-generic32 $(package)_config_opts_mips_linux=linux-generic32 $(package)_config_opts_powerpc_linux=linux-generic32 diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 562f4f7d6..1e5a74670 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -4,13 +4,17 @@ native_packages := native_ccache native_protobuf darwin_native_packages = native_biplist native_ds_store native_mac_alias darwin_packages = sodium-darwin ncurses readline -linux_packages = eudev ncurses readline unwind sodium +linux_packages = eudev ncurses readline sodium qt_packages = qt ifeq ($(build_tests),ON) packages += gtest endif +ifneq ($(host_arch),riscv64) + packages += unwind +endif + ifeq ($(host_os),mingw32) packages += icu4c packages += sodium diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index ee0407a5e..b748f5c55 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -98,6 +98,11 @@ elseif(ARCHITECTURE STREQUAL "aarch64") set(BUILD_64 ON) endif() +if(ARCHITECTURE STREQUAL "riscv64") + set(NO_AES ON) + set(ARCH "rv64imafdc") +endif() + if(ARCHITECTURE STREQUAL "i686" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") SET(LINUX_32 ON) SET(ARCH_ID "i386") diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h new file mode 100644 index 000000000..1fbba101e --- /dev/null +++ b/contrib/epee/include/byte_slice.h @@ -0,0 +1,145 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <string> +#include <vector> + +#include "span.h" + +namespace epee +{ + struct byte_slice_data; + + struct release_byte_slice + { + void operator()(byte_slice_data*) const noexcept; + }; + + /*! Inspired by slices in golang. Storage is thread-safe reference counted, + allowing for cheap copies or range selection on the bytes. The bytes + owned by this class are always immutable. + + The functions `operator=`, `take_slice` and `remove_prefix` may alter the + reference count for the backing store, which will invalidate pointers + previously returned if the reference count is zero. Be careful about + "caching" pointers in these circumstances. */ + class byte_slice + { + /* A custom reference count is used instead of shared_ptr because it allows + for an allocation optimization for the span constructor. This also + reduces the size of this class by one pointer. */ + std::unique_ptr<byte_slice_data, release_byte_slice> storage_; + span<const std::uint8_t> portion_; // within storage_ + + //! Internal use only; use to increase `storage` reference count. + byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept; + + struct adapt_buffer{}; + + template<typename T> + explicit byte_slice(const adapt_buffer, T&& buffer); + + public: + using value_type = std::uint8_t; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using pointer = const std::uint8_t*; + using const_pointer = const std::uint8_t*; + using reference = std::uint8_t; + using const_reference = std::uint8_t; + using iterator = pointer; + using const_iterator = const_pointer; + + //! Construct empty slice. + byte_slice() noexcept + : storage_(nullptr), portion_() + {} + + //! Construct empty slice + byte_slice(std::nullptr_t) noexcept + : byte_slice() + {} + + //! Scatter-gather (copy) multiple `sources` into a single allocated slice. + explicit byte_slice(std::initializer_list<span<const std::uint8_t>> sources); + + //! Convert `buffer` into a slice using one allocation for shared count. + explicit byte_slice(std::vector<std::uint8_t>&& buffer); + + //! Convert `buffer` into a slice using one allocation for shared count. + explicit byte_slice(std::string&& buffer); + + byte_slice(byte_slice&& source) noexcept; + ~byte_slice() noexcept = default; + + //! \note May invalidate previously retrieved pointers. + byte_slice& operator=(byte_slice&&) noexcept; + + //! \return A shallow (cheap) copy of the data from `this` slice. + byte_slice clone() const noexcept { return {storage_.get(), portion_}; } + + iterator begin() const noexcept { return portion_.begin(); } + const_iterator cbegin() const noexcept { return portion_.begin(); } + + iterator end() const noexcept { return portion_.end(); } + const_iterator cend() const noexcept { return portion_.end(); } + + bool empty() const noexcept { return storage_ == nullptr; } + const std::uint8_t* data() const noexcept { return portion_.data(); } + std::size_t size() const noexcept { return portion_.size(); } + + /*! Drop bytes from the beginning of `this` slice. + + \note May invalidate previously retrieved pointers. + \post `this->size() = this->size() - std::min(this->size(), max_bytes)` + \post `if (this->size() <= max_bytes) this->data() = nullptr` + \return Number of bytes removed. */ + std::size_t remove_prefix(std::size_t max_bytes) noexcept; + + /*! "Take" bytes from the beginning of `this` slice. + + \note May invalidate previously retrieved pointers. + \post `this->size() = this->size() - std::min(this->size(), max_bytes)` + \post `if (this->size() <= max_bytes) this->data() = nullptr` + \return Slice containing the bytes removed from `this` slice. */ + byte_slice take_slice(std::size_t max_bytes) noexcept; + + /*! Return a shallow (cheap) copy of a slice from `begin` and `end` offsets. + + \throw std::out_of_range If `end < begin`. + \throw std::out_of_range If `size() < end`. + \return Slice starting at `data() + begin` of size `end - begin`. */ + byte_slice get_slice(std::size_t begin, std::size_t end) const; + }; +} // epee + diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index e07e16d91..13747b0c8 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -45,6 +45,9 @@ #include "readline_buffer.h" #endif +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "console_handler" + namespace epee { class async_stdin_reader @@ -96,7 +99,7 @@ namespace epee res = true; } - if (!eos()) + if (!eos() && m_read_status != state_cancelled) m_read_status = state_init; return res; @@ -122,6 +125,14 @@ namespace epee } } + void cancel() + { + boost::unique_lock<boost::mutex> lock(m_response_mutex); + m_read_status = state_cancelled; + m_has_read_request = false; + m_response_cv.notify_one(); + } + private: bool start_read() { @@ -162,6 +173,9 @@ namespace epee while (m_run.load(std::memory_order_relaxed)) { + if (m_read_status == state_cancelled) + return false; + fd_set read_set; FD_ZERO(&read_set); FD_SET(stdin_fileno, &read_set); @@ -179,6 +193,9 @@ namespace epee #else while (m_run.load(std::memory_order_relaxed)) { + if (m_read_status == state_cancelled) + return false; + int retval = ::WaitForSingleObject(::GetStdHandle(STD_INPUT_HANDLE), 100); switch (retval) { @@ -219,7 +236,8 @@ reread: case rdln::full: break; } #else - std::getline(std::cin, line); + if (m_read_status != state_cancelled) + std::getline(std::cin, line); #endif read_ok = !std::cin.eof() && !std::cin.fail(); } @@ -303,7 +321,7 @@ eof: template<class chain_handler> bool run(chain_handler ch_handler, std::function<std::string(void)> prompt, const std::string& usage = "", std::function<void(void)> exit_handler = NULL) { - return run(prompt, usage, [&](const std::string& cmd) { return ch_handler(cmd); }, exit_handler); + return run(prompt, usage, [&](const boost::optional<std::string>& cmd) { return ch_handler(cmd); }, exit_handler); } void stop() @@ -312,6 +330,12 @@ eof: m_stdin_reader.stop(); } + void cancel() + { + m_cancel = true; + m_stdin_reader.cancel(); + } + void print_prompt() { std::string prompt = m_prompt(); @@ -360,18 +384,23 @@ eof: std::cout << std::endl; break; } + + if (m_cancel) + { + MDEBUG("Input cancelled"); + cmd_handler(boost::none); + m_cancel = false; + continue; + } if (!get_line_ret) { MERROR("Failed to read line."); } + string_tools::trim(command); LOG_PRINT_L2("Read command: " << command); - if (command.empty()) - { - continue; - } - else if(cmd_handler(command)) + if(cmd_handler(command)) { continue; } @@ -401,6 +430,7 @@ eof: private: async_stdin_reader m_stdin_reader; std::atomic<bool> m_running = {true}; + std::atomic<bool> m_cancel = {false}; std::function<std::string(void)> m_prompt; }; @@ -482,8 +512,16 @@ eof: class command_handler { public: typedef boost::function<bool (const std::vector<std::string> &)> callback; + typedef boost::function<bool (void)> empty_callback; typedef std::map<std::string, std::pair<callback, std::pair<std::string, std::string>>> lookup; + command_handler(): + m_unknown_command_handler([](const std::vector<std::string>&){return false;}), + m_empty_command_handler([](){return true;}), + m_cancel_handler([](){return true;}) + { + } + std::string get_usage() { std::stringstream ss; @@ -516,25 +554,45 @@ eof: #endif } + void set_unknown_command_handler(const callback& hndlr) + { + m_unknown_command_handler = hndlr; + } + + void set_empty_command_handler(const empty_callback& hndlr) + { + m_empty_command_handler = hndlr; + } + + void set_cancel_handler(const empty_callback& hndlr) + { + m_cancel_handler = hndlr; + } + bool process_command_vec(const std::vector<std::string>& cmd) { - if(!cmd.size()) - return false; + if(!cmd.size() || (cmd.size() == 1 && !cmd[0].size())) + return m_empty_command_handler(); auto it = m_command_handlers.find(cmd.front()); if(it == m_command_handlers.end()) - return false; + return m_unknown_command_handler(cmd); std::vector<std::string> cmd_local(cmd.begin()+1, cmd.end()); return it->second.first(cmd_local); } - bool process_command_str(const std::string& cmd) + bool process_command_str(const boost::optional<std::string>& cmd) { + if (!cmd) + return m_cancel_handler(); std::vector<std::string> cmd_v; - boost::split(cmd_v,cmd,boost::is_any_of(" "), boost::token_compress_on); + boost::split(cmd_v,*cmd,boost::is_any_of(" "), boost::token_compress_on); return process_command_vec(cmd_v); } private: lookup m_command_handlers; + callback m_unknown_command_handler; + empty_callback m_empty_command_handler; + empty_callback m_cancel_handler; }; /************************************************************************/ @@ -572,6 +630,11 @@ eof: { m_console_handler.print_prompt(); } + + void cancel_input() + { + m_console_handler.cancel(); + } }; ///* work around because of broken boost bind */ diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index c1aa0fe5f..3a2c5341d 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -49,10 +49,12 @@ #include <boost/asio/ssl.hpp> #include <boost/array.hpp> #include <boost/noncopyable.hpp> -#include <boost/shared_ptr.hpp> +#include <boost/shared_ptr.hpp> //! \TODO Convert to std::shared_ptr #include <boost/enable_shared_from_this.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/thread/thread.hpp> +#include <memory> +#include "byte_slice.h" #include "net_utils_base.h" #include "syncobj.h" #include "connection_basic.hpp" @@ -90,25 +92,24 @@ namespace net_utils public: typedef typename t_protocol_handler::connection_context t_connection_context; - struct shared_state : connection_basic_shared_state + struct shared_state : connection_basic_shared_state, t_protocol_handler::config_type { shared_state() - : connection_basic_shared_state(), pfilter(nullptr), config(), stop_signal_sent(false) + : connection_basic_shared_state(), t_protocol_handler::config_type(), pfilter(nullptr), stop_signal_sent(false) {} i_connection_filter* pfilter; - typename t_protocol_handler::config_type config; bool stop_signal_sent; }; /// Construct a connection with the given io_service. explicit connection( boost::asio::io_service& io_service, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, epee::net_utils::ssl_support_t ssl_support); explicit connection( boost::asio::ip::tcp::socket&& sock, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, epee::net_utils::ssl_support_t ssl_support); @@ -135,8 +136,7 @@ namespace net_utils private: //----------------- i_service_endpoint --------------------- - virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) - virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data + virtual bool do_send(byte_slice message); ///< (see do_send from i_service_endpoint) virtual bool send_done(); virtual bool close(); virtual bool call_run_once_service_io(); @@ -145,6 +145,8 @@ namespace net_utils virtual bool add_ref(); virtual bool release(); //------------------------------------------------------ + bool do_send_chunk(byte_slice chunk); ///< will send (or queue) a part of data. internal use only + boost::shared_ptr<connection<t_protocol_handler> > safe_shared_from_this(); bool shutdown(); /// Handle completion of a receive operation. @@ -227,8 +229,12 @@ namespace net_utils std::map<std::string, t_connection_type> server_type_map; void create_server_type_map(); - bool init_server(uint32_t port, const std::string address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); - bool init_server(const std::string port, const std::string& address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(uint32_t port, const std::string& address = "0.0.0.0", + uint32_t port_ipv6 = 0, const std::string& address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true, + ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(const std::string port, const std::string& address = "0.0.0.0", + const std::string port_ipv6 = "", const std::string address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true, + ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); /// Run the server's io_service loop. bool run_server(size_t threads_count, bool wait = true, const boost::thread::attributes& attrs = boost::thread::attributes()); @@ -265,10 +271,17 @@ namespace net_utils typename t_protocol_handler::config_type& get_config_object() { assert(m_state != nullptr); // always set in constructor - return m_state->config; + return *m_state; + } + + std::shared_ptr<typename t_protocol_handler::config_type> get_config_shared() + { + assert(m_state != nullptr); // always set in constructor + return {m_state}; } int get_binded_port(){return m_port;} + int get_binded_port_ipv6(){return m_port_ipv6;} long get_connections_count() const { @@ -339,11 +352,13 @@ namespace net_utils /// Run the server's io_service loop. bool worker_thread(); /// Handle completion of an asynchronous accept operation. - void handle_accept(const boost::system::error_code& e); + void handle_accept_ipv4(const boost::system::error_code& e); + void handle_accept_ipv6(const boost::system::error_code& e); + void handle_accept(const boost::system::error_code& e, bool ipv6 = false); bool is_thread_worker(); - const boost::shared_ptr<typename connection<t_protocol_handler>::shared_state> m_state; + const std::shared_ptr<typename connection<t_protocol_handler>::shared_state> m_state; /// The io_service used to perform asynchronous operations. struct worker @@ -360,11 +375,16 @@ namespace net_utils /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; + boost::asio::ip::tcp::acceptor acceptor_ipv6; epee::net_utils::network_address default_remote; std::atomic<bool> m_stop_signal_sent; uint32_t m_port; + uint32_t m_port_ipv6; std::string m_address; + std::string m_address_ipv6; + bool m_use_ipv6; + bool m_require_ipv4; std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; std::vector<boost::shared_ptr<boost::thread> > m_threads; @@ -376,6 +396,8 @@ namespace net_utils /// The next connection to be accepted connection_ptr new_connection_; + connection_ptr new_connection_ipv6; + boost::mutex connections_mutex; std::set<connection_ptr> connections_; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 0721366aa..12a87071a 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -68,7 +68,7 @@ namespace epee namespace net_utils { template<typename T> - T& check_and_get(boost::shared_ptr<T>& ptr) + T& check_and_get(std::shared_ptr<T>& ptr) { CHECK_AND_ASSERT_THROW_MES(bool(ptr), "shared_state cannot be null"); return *ptr; @@ -81,7 +81,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> connection<t_protocol_handler>::connection( boost::asio::io_service& io_service, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, ssl_support_t ssl_support ) @@ -91,13 +91,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> connection<t_protocol_handler>::connection( boost::asio::ip::tcp::socket&& sock, - boost::shared_ptr<shared_state> state, + std::shared_ptr<shared_state> state, t_connection_type connection_type, ssl_support_t ssl_support ) : connection_basic(std::move(sock), state, ssl_support), - m_protocol_handler(this, check_and_get(state).config, context), + m_protocol_handler(this, check_and_get(state), context), buffer_ssl_init_fill(0), m_connection_type( connection_type ), m_throttle_speed_in("speed_in", "throttle_speed_in"), @@ -145,10 +145,18 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::system::error_code ec; auto remote_ep = socket().remote_endpoint(ec); CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value()); - CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4(), false, "IPv6 not supported here"); + CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4() || remote_ep.address().is_v6(), false, "only IPv4 and IPv6 supported here"); - const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())}; - return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + if (remote_ep.address().is_v4()) + { + const unsigned long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); + return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + } + else + { + const auto ip_{remote_ep.address().to_v6()}; + return start(is_income, is_multithreaded, ipv6_network_address{ip_, remote_ep.port()}); + } CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false); } //--------------------------------------------------------------------------------- @@ -370,7 +378,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(!recv_res) { //_info("[sock " << socket().native_handle() << "] protocol_want_close"); - //some error in protocol, protocol handler ask to close connection boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); bool do_shutdown = false; @@ -512,7 +519,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) { + bool connection<t_protocol_handler>::do_send(byte_slice message) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -521,6 +528,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (m_was_shutdown) return false; // TODO avoid copy + std::uint8_t const* const message_data = message.data(); + const std::size_t message_size = message.size(); + const double factor = 32; // TODO config typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) ); @@ -530,13 +540,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) CHECK_AND_ASSERT_MES(! (chunksize_max<0), false, "Negative chunksize_max" ); // make sure it is unsigned before removin sign with cast: long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ; - if (allow_split && (cb > chunksize_max_unsigned)) { + if (allow_split && (message_size > chunksize_max_unsigned)) { { // LOCK: chunking epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** - MDEBUG("do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr); - t_safe all = cb; // all bytes to send - t_safe pos = 0; // current sending position + MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<message_data); // 01234567890 // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 @@ -546,40 +554,25 @@ PRAGMA_WARNING_DISABLE_VS(4355) // char* buf = new char[ bufsize ]; bool all_ok = true; - while (pos < all) { - t_safe lenall = all-pos; // length from here to end - t_safe len = std::min( chunksize_good , lenall); // take a smaller part - CHECK_AND_ASSERT_MES(len<=chunksize_good, false, "len too large"); - // pos=8; len=4; all=10; len=3; - - CHECK_AND_ASSERT_MES(! (len<0), false, "negative len"); // check before we cast away sign: - unsigned long long int len_unsigned = static_cast<long long int>( len ); - CHECK_AND_ASSERT_MES(len>0, false, "len not strictly positive"); // (redundant) - CHECK_AND_ASSERT_MES(len_unsigned < std::numeric_limits<size_t>::max(), false, "Invalid len_unsigned"); // yeap we want strong < then max size, to be sure - - void *chunk_start = ((char*)ptr) + pos; - MDEBUG("chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos); - CHECK_AND_ASSERT_MES(chunk_start >= ptr, false, "Pointer wraparound"); // not wrapped around address? - //std::memcpy( (void*)buf, chunk_start, len); - - MDEBUG("part of " << lenall << ": pos="<<pos << " len="<<len); - - bool ok = do_send_chunk(chunk_start, len); // <====== *** + while (!message.empty()) { + byte_slice chunk = message.take_slice(chunksize_good); + + MDEBUG("chunk_start="<<chunk.data()<<" ptr="<<message_data<<" pos="<<(chunk.data() - message_data)); + MDEBUG("part of " << message.size() << ": pos="<<(chunk.data() - message_data) << " len="<<chunk.size()); + + bool ok = do_send_chunk(std::move(chunk)); // <====== *** all_ok = all_ok && ok; if (!all_ok) { - MDEBUG("do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr); + MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<message_data); MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless " - << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb); + << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << message_size); return false; // partial failure in sending } - pos = pos+len; - CHECK_AND_ASSERT_MES(pos >0, false, "pos <= 0"); - // (in catch block, or uniq pointer) delete buf; } // each chunk - MDEBUG("do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr); + MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<message_data); MDEBUG("do_send() m_connection_type = " << m_connection_type); @@ -587,7 +580,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } // LOCK: chunking } // a big block (to be chunked) - all chunks else { // small block - return do_send_chunk(ptr,cb); // just send as 1 big chunk + return do_send_chunk(std::move(message)); // just send as 1 big chunk } CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); @@ -595,7 +588,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb) + bool connection<t_protocol_handler>::do_send_chunk(byte_slice chunk) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -607,7 +600,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) double current_speed_up; { CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); - m_throttle_speed_out.handle_trafic_exact(cb); + m_throttle_speed_out.handle_trafic_exact(chunk.size()); current_speed_up = m_throttle_speed_out.get_current_speed(); } context.m_current_speed_up = current_speed_up; @@ -615,7 +608,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //_info("[sock " << socket().native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); - context.m_send_cnt += cb; + context.m_send_cnt += chunk.size(); //some data should be wrote to stream //request complete @@ -636,7 +629,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) }*/ long int ms = 250 + (rand()%50); - MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep + MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<chunk.size()); // XXX debug sleep m_send_que_lock.unlock(); boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); m_send_que_lock.lock(); @@ -649,12 +642,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) } } - m_send_que.resize(m_send_que.size()+1); - m_send_que.back().assign((const char*)ptr, cb); - + m_send_que.push_back(std::move(chunk)); + if(m_send_que.size() > 1) { // active operation should be in progress, nothing to do, just wait last operation callback - auto size_now = cb; + auto size_now = m_send_que.back().size(); MDEBUG("do_send_chunk() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size()); //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function @@ -672,7 +664,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) auto size_now = m_send_que.front().size(); MDEBUG("do_send_chunk() NOW SENSD: packet="<<size_now<<" B"); if (speed_limit_is_enabled()) - do_send_handler_write( ptr , size_now ); // (((H))) + do_send_handler_write( m_send_que.back().data(), m_send_que.back().size() ); // (((H))) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size"); reset_timer(get_default_timeout(), false); @@ -900,16 +892,18 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> boosted_tcp_server<t_protocol_handler>::boosted_tcp_server( t_connection_type connection_type ) : - m_state(boost::make_shared<typename connection<t_protocol_handler>::shared_state>()), + m_state(std::make_shared<typename connection<t_protocol_handler>::shared_state>()), m_io_service_local_instance(new worker()), io_service_(m_io_service_local_instance->io_service), acceptor_(io_service_), + acceptor_ipv6(io_service_), default_remote(), m_stop_signal_sent(false), m_port(0), m_threads_count(0), m_thread_index(0), m_connection_type( connection_type ), - new_connection_() + new_connection_(), + new_connection_ipv6() { create_server_type_map(); m_thread_name_prefix = "NET"; @@ -917,15 +911,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_connection_type connection_type) : - m_state(boost::make_shared<typename connection<t_protocol_handler>::shared_state>()), + m_state(std::make_shared<typename connection<t_protocol_handler>::shared_state>()), io_service_(extarnal_io_service), acceptor_(io_service_), + acceptor_ipv6(io_service_), default_remote(), m_stop_signal_sent(false), m_port(0), m_threads_count(0), m_thread_index(0), m_connection_type(connection_type), - new_connection_() + new_connection_(), + new_connection_ipv6() { create_server_type_map(); m_thread_name_prefix = "NET"; @@ -947,29 +943,92 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address, ssl_options_t ssl_options) + bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string& address, + uint32_t port_ipv6, const std::string& address_ipv6, bool use_ipv6, bool require_ipv4, + ssl_options_t ssl_options) { TRY_ENTRY(); m_stop_signal_sent = false; m_port = port; + m_port_ipv6 = port_ipv6; m_address = address; + m_address_ipv6 = address_ipv6; + m_use_ipv6 = use_ipv6; + m_require_ipv4 = require_ipv4; + if (ssl_options) m_state->configure_ssl(std::move(ssl_options)); - // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); - acceptor_.open(endpoint.protocol()); - acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); - acceptor_.bind(endpoint); - acceptor_.listen(); - boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); - m_port = binded_endpoint.port(); - MDEBUG("start accept"); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + + std::string ipv4_failed = ""; + std::string ipv6_failed = ""; + try + { + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_.open(endpoint.protocol()); + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_.bind(endpoint); + acceptor_.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); + m_port = binded_endpoint.port(); + MDEBUG("start accept (IPv4)"); + new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); + acceptor_.async_accept(new_connection_->socket(), + boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4, this, + boost::asio::placeholders::error)); + } + catch (const std::exception &e) + { + ipv4_failed = e.what(); + } + + if (ipv4_failed != "") + { + MERROR("Failed to bind IPv4: " << ipv4_failed); + if (require_ipv4) + { + throw std::runtime_error("Failed to bind IPv4 (set to required)"); + } + } + + if (use_ipv6) + { + try + { + if (port_ipv6 == 0) port_ipv6 = port; // default arg means bind to same port as ipv4 + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address_ipv6, boost::lexical_cast<std::string>(port_ipv6), boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_ipv6.open(endpoint.protocol()); + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + acceptor_ipv6.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_ipv6.set_option(boost::asio::ip::v6_only(true)); + acceptor_ipv6.bind(endpoint); + acceptor_ipv6.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_ipv6.local_endpoint(); + m_port_ipv6 = binded_endpoint.port(); + MDEBUG("start accept (IPv6)"); + new_connection_ipv6.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); + acceptor_ipv6.async_accept(new_connection_ipv6->socket(), + boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6, this, + boost::asio::placeholders::error)); + } + catch (const std::exception &e) + { + ipv6_failed = e.what(); + } + } + + if (use_ipv6 && ipv6_failed != "") + { + MERROR("Failed to bind IPv6: " << ipv6_failed); + if (ipv4_failed != "") + { + throw std::runtime_error("Failed to bind IPv4 and IPv6"); + } + } return true; } @@ -988,15 +1047,23 @@ PRAGMA_WARNING_DISABLE_VS(4355) PUSH_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) template<class t_protocol_handler> - bool boosted_tcp_server<t_protocol_handler>::init_server(const std::string port, const std::string& address, ssl_options_t ssl_options) + bool boosted_tcp_server<t_protocol_handler>::init_server(const std::string port, const std::string& address, + const std::string port_ipv6, const std::string address_ipv6, bool use_ipv6, bool require_ipv4, + ssl_options_t ssl_options) { uint32_t p = 0; + uint32_t p_ipv6 = 0; if (port.size() && !string_tools::get_xtype_from_string(p, port)) { MERROR("Failed to convert port no = " << port); return false; } - return this->init_server(p, address, std::move(ssl_options)); + + if (port_ipv6.size() && !string_tools::get_xtype_from_string(p_ipv6, port_ipv6)) { + MERROR("Failed to convert port no = " << port_ipv6); + return false; + } + return this->init_server(p, address, p_ipv6, address_ipv6, use_ipv6, require_ipv4, std::move(ssl_options)); } POP_WARNINGS //--------------------------------------------------------------------------------- @@ -1088,7 +1155,7 @@ POP_WARNINGS { //some problems with the listening socket ?.. _dbg1("Net service stopped without stop request, restarting..."); - if(!this->init_server(m_port, m_address)) + if(!this->init_server(m_port, m_address, m_port_ipv6, m_address_ipv6, m_use_ipv6, m_require_ipv4)) { _dbg1("Reiniting service failed, exit."); return false; @@ -1154,29 +1221,52 @@ POP_WARNINGS } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e) + void boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4(const boost::system::error_code& e) + { + this->handle_accept(e, false); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6(const boost::system::error_code& e) + { + this->handle_accept(e, true); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e, bool ipv6) { MDEBUG("handle_accept"); + + boost::asio::ip::tcp::acceptor* current_acceptor = &acceptor_; + connection_ptr* current_new_connection = &new_connection_; + auto accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4; + if (ipv6) + { + current_acceptor = &acceptor_ipv6; + current_new_connection = &new_connection_ipv6; + accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6; + } + try { if (!e) { - if (m_connection_type == e_connection_type_RPC) { - const char *ssl_message = "unknown"; - switch (new_connection_->get_ssl_support()) - { - case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; - case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; - case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; - } - MDEBUG("New server for RPC connections, SSL " << ssl_message); - new_connection_->setRpcStation(); // hopefully this is not needed actually - } - connection_ptr conn(std::move(new_connection_)); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support())); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + if (m_connection_type == e_connection_type_RPC) { + const char *ssl_message = "unknown"; + switch ((*current_new_connection)->get_ssl_support()) + { + case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; + } + MDEBUG("New server for RPC connections, SSL " << ssl_message); + (*current_new_connection)->setRpcStation(); // hopefully this is not needed actually + } + connection_ptr conn(std::move((*current_new_connection))); + (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support())); + current_acceptor->async_accept((*current_new_connection)->socket(), + boost::bind(accept_function_pointer, this, + boost::asio::placeholders::error)); boost::asio::socket_base::keep_alive opt(true); conn->socket().set_option(opt); @@ -1208,10 +1298,10 @@ POP_WARNINGS assert(m_state != nullptr); // always set in constructor _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_state->sock_count); misc_utils::sleep_no_w(100); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, new_connection_->get_ssl_support())); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, (*current_new_connection)->get_ssl_support())); + current_acceptor->async_accept((*current_new_connection)->socket(), + boost::bind(accept_function_pointer, this, + boost::asio::placeholders::error)); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -1345,23 +1435,84 @@ POP_WARNINGS epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - ////////////////////////////////////////////////////////////////////////// + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::system::error_code resolve_error; + boost::asio::ip::tcp::resolver::iterator iterator; + try + { + //resolving ipv4 address as ipv6 throws, catch here and move on + iterator = resolver.resolve(query, resolve_error); + } + catch (const boost::system::system_error& e) + { + if (!m_use_ipv6 || (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again)) + { + throw; + } + try_ipv6 = true; + } + catch (...) + { + throw; + } + + std::string bind_ip_to_use; + boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - _erro("Failed to resolve " << adr); - return false; + if (!m_use_ipv6) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + try_ipv6 = true; + MINFO("Resolving address as IPv4 failed, trying IPv6"); + } } - ////////////////////////////////////////////////////////////////////////// + else + { + bind_ip_to_use = bind_ip; + } + + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + + iterator = resolver.resolve(query6, resolve_error); + + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + if (bind_ip == "0.0.0.0") + { + bind_ip_to_use = "::"; + } + else + { + bind_ip_to_use = ""; + } + } + + } + + LOG_ERROR("Trying connect to " << adr << ":" << port << ", bind_ip = " << bind_ip_to_use); //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, ssl_support); + auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout, ssl_support); if (try_connect_result == CONNECT_FAILURE) return false; if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect && try_connect_result == CONNECT_NO_SSL) @@ -1369,7 +1520,7 @@ POP_WARNINGS // we connected, but could not connect with SSL, try without MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); new_connection_l->disable_ssl(); - try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); if (try_connect_result != CONNECT_SUCCESS) return false; } @@ -1409,17 +1560,59 @@ POP_WARNINGS epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - ////////////////////////////////////////////////////////////////////////// + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::system::error_code resolve_error; + boost::asio::ip::tcp::resolver::iterator iterator; + try + { + //resolving ipv4 address as ipv6 throws, catch here and move on + iterator = resolver.resolve(query, resolve_error); + } + catch (const boost::system::system_error& e) + { + if (!m_use_ipv6 || (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again)) + { + throw; + } + try_ipv6 = true; + } + catch (...) + { + throw; + } + boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - _erro("Failed to resolve " << adr); - return false; + if (!try_ipv6) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + MINFO("Resolving address as IPv4 failed, trying IPv6"); + } + } + + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + + iterator = resolver.resolve(query6, resolve_error); + + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } } - ////////////////////////////////////////////////////////////////////////// + + boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); sock_.open(remote_endpoint.protocol()); diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp index 2acc6cdda..2f60f7604 100644 --- a/contrib/epee/include/net/connection_basic.hpp +++ b/contrib/epee/include/net/connection_basic.hpp @@ -49,6 +49,7 @@ #include <boost/asio.hpp> #include <boost/asio/ssl.hpp> +#include "byte_slice.h" #include "net/net_utils_base.h" #include "net/net_ssl.h" #include "syncobj.h" @@ -99,7 +100,7 @@ class connection_basic_pimpl; // PIMPL for this class class connection_basic { // not-templated base class for rapid developmet of some code parts // beware of removing const, net_utils::connection is sketchily doing a cast to prevent storing ptr twice - const boost::shared_ptr<connection_basic_shared_state> m_state; + const std::shared_ptr<connection_basic_shared_state> m_state; public: std::unique_ptr< connection_basic_pimpl > mI; // my Implementation @@ -108,7 +109,7 @@ class connection_basic { // not-templated base class for rapid developmet of som volatile uint32_t m_want_close_connection; std::atomic<bool> m_was_shutdown; critical_section m_send_que_lock; - std::list<std::string> m_send_que; + std::deque<byte_slice> m_send_que; volatile bool m_is_multithreaded; /// Strand to ensure the connection's handlers are not called concurrently. boost::asio::io_service::strand strand_; @@ -118,8 +119,8 @@ class connection_basic { // not-templated base class for rapid developmet of som public: // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator - connection_basic(boost::asio::ip::tcp::socket&& socket, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); - connection_basic(boost::asio::io_service &io_service, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); + connection_basic(boost::asio::ip::tcp::socket&& socket, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); + connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support); virtual ~connection_basic() noexcept(false); diff --git a/contrib/epee/include/net/enums.h b/contrib/epee/include/net/enums.h index 078a4b274..2f27d07f9 100644 --- a/contrib/epee/include/net/enums.h +++ b/contrib/epee/include/net/enums.h @@ -49,7 +49,7 @@ namespace net_utils { invalid = 0, public_ = 1, // public is keyword - i2p = 2, + i2p = 2, // order from here changes priority of selection for origin TXes tor = 3 }; diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index ae8e43477..19bdf4ff0 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -577,6 +577,10 @@ namespace net_utils if (query_info.m_http_method != http::http_method_options) { res = handle_request(query_info, response); + if (response.m_response_code == 500) + { + m_want_close = true; // close on all "Internal server error"s + } } else { @@ -587,11 +591,12 @@ namespace net_utils std::string response_data = get_response_header(response); //LOG_PRINT_L0("HTTP_SEND: << \r\n" << response_data + response.m_body); - LOG_PRINT_L3("HTTP_RESPONSE_HEAD: << \r\n" << response_data); - - m_psnd_hndlr->do_send((void*)response_data.data(), response_data.size()); + LOG_PRINT_L3("HTTP_RESPONSE_HEAD: << \r\n" << response_data); + if ((response.m_body.size() && (query_info.m_http_method != http::http_method_head)) || (query_info.m_http_method == http::http_method_options)) - m_psnd_hndlr->do_send((void*)response.m_body.data(), response.m_body.size()); + response_data += response.m_body; + + m_psnd_hndlr->do_send(byte_slice{std::move(response_data)}); m_psnd_hndlr->send_done(); return res; } diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 4b2053091..07ed8157b 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -71,7 +71,7 @@ MINFO(m_conn_context << "calling " << s_pattern); \ if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ { \ - LOG_ERROR("Failed to " << #callback_f << "()"); \ + MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ @@ -99,7 +99,7 @@ MINFO(m_conn_context << "calling " << s_pattern); \ if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ { \ - LOG_ERROR("Failed to " << #callback_f << "()"); \ + MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ diff --git a/contrib/epee/include/net/http_server_impl_base.h b/contrib/epee/include/net/http_server_impl_base.h index fc2dcbf67..6cd19f17b 100644 --- a/contrib/epee/include/net/http_server_impl_base.h +++ b/contrib/epee/include/net/http_server_impl_base.h @@ -57,6 +57,7 @@ namespace epee {} bool init(std::function<void(size_t, uint8_t*)> rng, const std::string& bind_port = "0", const std::string& bind_ip = "0.0.0.0", + const std::string& bind_ipv6_address = "::", bool use_ipv6 = false, bool require_ipv4 = true, std::vector<std::string> access_control_origins = std::vector<std::string>(), boost::optional<net_utils::http::login> user = boost::none, net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect) @@ -75,8 +76,12 @@ namespace epee m_net_server.get_config_object().m_user = std::move(user); - MGINFO("Binding on " << bind_ip << ":" << bind_port); - bool res = m_net_server.init_server(bind_port, bind_ip, std::move(ssl_options)); + MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port); + if (use_ipv6) + { + MGINFO("Binding on " << bind_ipv6_address << " (IPv6):" << bind_port); + } + bool res = m_net_server.init_server(bind_port, bind_ip, bind_port, bind_ipv6_address, use_ipv6, require_ipv4, std::move(ssl_options)); if(!res) { LOG_ERROR("Failed to bind server"); diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index a88a1eb49..f9b6f9a81 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -29,7 +29,11 @@ #ifndef _LEVIN_BASE_H_ #define _LEVIN_BASE_H_ +#include <cstdint> + +#include "byte_slice.h" #include "net_utils_base.h" +#include "span.h" #define LEVIN_SIGNATURE 0x0101010101012101LL //Bender's nightmare @@ -72,6 +76,8 @@ namespace levin #define LEVIN_PACKET_REQUEST 0x00000001 #define LEVIN_PACKET_RESPONSE 0x00000002 +#define LEVIN_PACKET_BEGIN 0x00000004 +#define LEVIN_PACKET_END 0x00000008 #define LEVIN_PROTOCOL_VER_0 0 @@ -118,9 +124,30 @@ namespace levin } } + //! \return Intialized levin header. + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; + + //! \return A levin notification message. + byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); + + /*! Generate a dummy levin message. + \param noise_bytes Total size of the returned `byte_slice`. + \return `nullptr` if `noise_size` is smaller than the levin header. + Otherwise, a dummy levin message. */ + byte_slice make_noise_notify(std::size_t noise_bytes); + + /*! Generate 1+ levin messages that are identical to the noise message size. + + \param noise Each levin message will be identical to the size of this + message. The bytes from this message will be used for padding. + \return `nullptr` if `noise.size()` is less than the levin header size. + Otherwise, a levin notification message OR 2+ levin fragment messages. + Each message is `noise.size()` in length. */ + byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); } } #endif //_LEVIN_BASE_H_ + diff --git a/contrib/epee/include/net/levin_protocol_handler.h b/contrib/epee/include/net/levin_protocol_handler.h index 791766762..c510cfd79 100644 --- a/contrib/epee/include/net/levin_protocol_handler.h +++ b/contrib/epee/include/net/levin_protocol_handler.h @@ -157,10 +157,9 @@ namespace levin m_current_head.m_return_code = m_config.m_pcommands_handler->invoke(m_current_head.m_command, buff_to_invoke, return_buff, m_conn_context); m_current_head.m_cb = return_buff.size(); m_current_head.m_have_to_return_data = false; - std::string send_buff((const char*)&m_current_head, sizeof(m_current_head)); - send_buff += return_buff; - if(!m_psnd_hndlr->do_send(send_buff.data(), send_buff.size())) + return_buff.insert(0, (const char*)&m_current_head, sizeof(m_current_head)); + if(!m_psnd_hndlr->do_send(byte_slice{std::move(return_buff)})) return false; } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 116b3ace1..208911e1a 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -32,6 +32,7 @@ #include <boost/smart_ptr/make_shared.hpp> #include <atomic> +#include <deque> #include "levin_base.h" #include "buffer.h" @@ -91,6 +92,7 @@ public: int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); + int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); bool request_callback(boost::uuids::uuid connection_id); @@ -99,6 +101,8 @@ public: template<class callback_t> bool for_connection(const boost::uuids::uuid &connection_id, const callback_t &cb); size_t get_connections_count(); + size_t get_out_connections_count(); + size_t get_in_connections_count(); void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL); async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) @@ -115,6 +119,22 @@ public: template<class t_connection_context = net_utils::connection_context_base> class async_protocol_handler { + std::string m_fragment_buffer; + + bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + { + const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); + if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + return false; + + MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb + << ", flags" << head.m_flags + << ", r?=" << head.m_have_to_return_data + <<", cmd = " << head.m_command + << ", ver=" << head.m_protocol_version); + return true; + } + public: typedef t_connection_context connection_context; typedef async_protocol_handler_config<t_connection_context> config_type; @@ -134,7 +154,6 @@ public: critical_section m_local_inv_buff_lock; std::string m_local_inv_buff; - critical_section m_send_lock; critical_section m_call_lock; volatile uint32_t m_wait_count; @@ -374,7 +393,12 @@ public: return false; } - if(m_cache_in_buffer.size() + cb > m_config.m_max_packet_size) + // these should never fail, but do runtime check for safety + CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()"); + CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()"); + + // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public + if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size()) { MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size << ", packet received " << m_cache_in_buffer.size() + cb @@ -406,8 +430,38 @@ public: } break; } + { + std::string temp{}; epee::span<const uint8_t> buff_to_invoke = m_cache_in_buffer.carve((std::string::size_type)m_current_head.m_cb); + m_state = stream_state_head; + + // abstract_tcp_server2.h manages max bandwidth for a p2p link + if (!(m_current_head.m_flags & (LEVIN_PACKET_REQUEST | LEVIN_PACKET_RESPONSE))) + { + // special noise/fragment command + static constexpr const uint32_t both_flags = (LEVIN_PACKET_BEGIN | LEVIN_PACKET_END); + if ((m_current_head.m_flags & both_flags) == both_flags) + break; // noise message, skip to next message + + if (m_current_head.m_flags & LEVIN_PACKET_BEGIN) + m_fragment_buffer.clear(); + + m_fragment_buffer.append(reinterpret_cast<const char*>(buff_to_invoke.data()), buff_to_invoke.size()); + if (!(m_current_head.m_flags & LEVIN_PACKET_END)) + break; // skip to next message + + if (m_fragment_buffer.size() < sizeof(bucket_head2)) + { + MERROR(m_connection_context << "Fragmented data too small for levin header"); + return false; + } + + temp = std::move(m_fragment_buffer); + m_fragment_buffer.clear(); + std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2)); + buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)}; + } bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE); @@ -456,43 +510,33 @@ public: if(m_current_head.m_have_to_return_data) { std::string return_buff; - m_current_head.m_return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, - buff_to_invoke, - return_buff, - m_connection_context); - m_current_head.m_cb = return_buff.size(); - m_current_head.m_have_to_return_data = false; - m_current_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; - m_current_head.m_flags = LEVIN_PACKET_RESPONSE; -#if BYTE_ORDER == LITTLE_ENDIAN - std::string send_buff((const char*)&m_current_head, sizeof(m_current_head)); -#else - bucket_head2 head = m_current_head; - head.m_signature = SWAP64LE(head.m_signature); - head.m_cb = SWAP64LE(head.m_cb); - head.m_command = SWAP32LE(head.m_command); - head.m_return_code = SWAP32LE(head.m_return_code); - head.m_flags = SWAP32LE(head.m_flags); - head.m_protocol_version = SWAP32LE(head.m_protocol_version); - std::string send_buff((const char*)&head, sizeof(head)); -#endif - send_buff += return_buff; - CRITICAL_REGION_BEGIN(m_send_lock); - if(!m_pservice_endpoint->do_send(send_buff.data(), send_buff.size())) + const uint32_t return_code = m_config.m_pcommands_handler->invoke( + m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + ); + + bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); + head.m_return_code = SWAP32LE(return_code); + return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head)); + + if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)})) return false; - CRITICAL_REGION_END(); - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << m_current_head.m_cb - << ", flags" << m_current_head.m_flags - << ", r?=" << m_current_head.m_have_to_return_data - <<", cmd = " << m_current_head.m_command - << ", ver=" << m_current_head.m_protocol_version); + + MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb + << ", flags" << head.m_flags + << ", r?=" << head.m_have_to_return_data + <<", cmd = " << head.m_command + << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); } + // reuse small buffer + if (!temp.empty() && temp.capacity() <= 64 * 1024) + { + temp.clear(); + m_fragment_buffer = std::move(temp); + } } - m_state = stream_state_head; break; case stream_state_head: { @@ -582,26 +626,10 @@ public: break; } - bucket_head2 head = {0}; - head.m_signature = SWAP64LE(LEVIN_SIGNATURE); - head.m_cb = SWAP64LE(in_buff.size()); - head.m_have_to_return_data = true; - - head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); - head.m_command = SWAP32LE(command); - head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); - CRITICAL_REGION_BEGIN(m_send_lock); - CRITICAL_REGION_LOCAL1(m_invoke_response_handlers_lock); - if(!m_pservice_endpoint->do_send(&head, sizeof(head))) - { - LOG_ERROR_CC(m_connection_context, "Failed to do_send"); - err_code = LEVIN_ERROR_CONNECTION; - break; - } + CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock); - if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) + if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -618,7 +646,7 @@ public: if (LEVIN_OK != err_code) { - epee::span<const uint8_t> stub_buff{(const uint8_t*)"", 0}; + epee::span<const uint8_t> stub_buff = nullptr; // Never call callback inside critical section, that can cause deadlock cb(err_code, stub_buff, m_connection_context); return false; @@ -640,35 +668,13 @@ public: if(m_deletion_initiated) return LEVIN_ERROR_CONNECTION_DESTROYED; - bucket_head2 head = {0}; - head.m_signature = SWAP64LE(LEVIN_SIGNATURE); - head.m_cb = SWAP64LE(in_buff.size()); - head.m_have_to_return_data = true; - - head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); - head.m_command = SWAP32LE(command); - head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); - CRITICAL_REGION_BEGIN(m_send_lock); - if(!m_pservice_endpoint->do_send(&head, sizeof(head))) - { - LOG_ERROR_CC(m_connection_context, "Failed to do_send"); - return LEVIN_ERROR_CONNECTION; - } - if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) + if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) { - LOG_ERROR_CC(m_connection_context, "Failed to do_send"); + LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; } - CRITICAL_REGION_END(); - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", f=" << head.m_flags - << ", r?=" << head.m_have_to_return_data - << ", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); uint64_t ticks_start = misc_utils::get_tick_count(); size_t prev_size = 0; @@ -714,33 +720,38 @@ public: if(m_deletion_initiated) return LEVIN_ERROR_CONNECTION_DESTROYED; - bucket_head2 head = {0}; - head.m_signature = SWAP64LE(LEVIN_SIGNATURE); - head.m_have_to_return_data = false; - head.m_cb = SWAP64LE(in_buff.size()); - - head.m_command = SWAP32LE(command); - head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); - head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); - CRITICAL_REGION_BEGIN(m_send_lock); - if(!m_pservice_endpoint->do_send(&head, sizeof(head))) + if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) { - LOG_ERROR_CC(m_connection_context, "Failed to do_send()"); + LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); return -1; } - if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) + return 1; + } + + /*! Sends `message` without adding a levin header. The message must have + been created with `make_notify`, `make_noise_notify` or + `make_fragmented_notify`. See additional instructions for + `make_fragmented_notify`. + + \return 1 on success */ + int send(byte_slice message) + { + const misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler::finish_outer_call, this) + ); + + if(m_deletion_initiated) + return LEVIN_ERROR_CONNECTION_DESTROYED; + + const std::size_t length = message.size(); + if (!m_pservice_endpoint->do_send(std::move(message))) { - LOG_ERROR_CC(m_connection_context, "Failed to do_send()"); + LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - CRITICAL_REGION_END(); - LOG_DEBUG_CC(m_connection_context, "LEVIN_PACKET_SENT. [len=" << head.m_cb << - ", f=" << head.m_flags << - ", r?=" << head.m_have_to_return_data << - ", cmd = " << head.m_command << - ", ver=" << head.m_protocol_version); + MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -882,6 +893,28 @@ size_t async_protocol_handler_config<t_connection_context>::get_connections_coun } //------------------------------------------------------------------------------------------ template<class t_connection_context> +size_t async_protocol_handler_config<t_connection_context>::get_out_connections_count() +{ + CRITICAL_REGION_LOCAL(m_connects_lock); + size_t count = 0; + for (const auto &c: m_connects) + if (!c.second->m_connection_context.m_is_income) + ++count; + return count; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> +size_t async_protocol_handler_config<t_connection_context>::get_in_connections_count() +{ + CRITICAL_REGION_LOCAL(m_connects_lock); + size_t count = 0; + for (const auto &c: m_connects) + if (c.second->m_connection_context.m_is_income) + ++count; + return count; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*)) { if (m_pcommands_handler && m_pcommands_handler_destroy) @@ -899,6 +932,14 @@ int async_protocol_handler_config<t_connection_context>::notify(int command, con } //------------------------------------------------------------------------------------------ template<class t_connection_context> +int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) +{ + async_protocol_handler<t_connection_context>* aph; + int r = find_and_lock_connection(connection_id, aph); + return LEVIN_OK == r ? aph->send(std::move(message)) : 0; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> bool async_protocol_handler_config<t_connection_context>::close(boost::uuids::uuid connection_id) { CRITICAL_REGION_LOCAL(m_connects_lock); diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index 52c5855b9..ce74e1cd3 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -27,10 +27,38 @@ #pragma once +#include <string> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/asio/ip/address_v6.hpp> + namespace epee { namespace net_utils { + + inline + bool is_ipv6_local(const std::string& ip) + { + auto addr = boost::asio::ip::address_v6::from_string(ip); + + // ipv6 link-local unicast addresses are fe80::/10 + bool is_link_local = addr.is_link_local(); + + auto addr_bytes = addr.to_bytes(); + + // ipv6 unique local unicast addresses start with fc00::/7 -- (fcXX or fdXX) + bool is_unique_local_unicast = (addr_bytes[0] == 0xfc || addr_bytes[0] == 0xfd); + + return is_link_local || is_unique_local_unicast; + } + + inline + bool is_ipv6_loopback(const std::string& ip) + { + // ipv6 loopback is ::1 + return boost::asio::ip::address_v6::from_string(ip).is_loopback(); + } + inline bool is_ip_local(uint32_t ip) { diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index e315555fc..2b02eafa4 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -31,6 +31,7 @@ //#include <Winsock2.h> //#include <Ws2tcpip.h> +#include <atomic> #include <string> #include <boost/version.hpp> #include <boost/asio/io_service.hpp> @@ -154,7 +155,7 @@ namespace net_utils } inline - try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, epee::net_utils::ssl_support_t ssl_support) + try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) { m_deadline.expires_from_now(timeout); boost::unique_future<boost::asio::ip::tcp::socket> connection = m_connector(addr, port, m_deadline); @@ -174,11 +175,11 @@ namespace net_utils m_connected = true; m_deadline.expires_at(std::chrono::steady_clock::time_point::max()); // SSL Options - if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr)) { - if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) + if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { boost::system::error_code ignored_ec; m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); @@ -217,7 +218,7 @@ namespace net_utils // Get a list of endpoints corresponding to the server name. - try_connect_result_t try_connect_result = try_connect(addr, port, timeout, m_ssl_options.support); + try_connect_result_t try_connect_result = try_connect(addr, port, timeout); if (try_connect_result == CONNECT_FAILURE) return false; if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) @@ -226,7 +227,7 @@ namespace net_utils { MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); m_ssl_options.support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; - if (try_connect(addr, port, timeout, m_ssl_options.support) != CONNECT_SUCCESS) + if (try_connect(addr, port, timeout) != CONNECT_SUCCESS) return false; } } @@ -562,7 +563,7 @@ namespace net_utils { m_deadline.cancel(); boost::system::error_code ec; - if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) + if(m_ssl_options) shutdown_ssl(); m_ssl_socket->next_layer().cancel(ec); if(ec) diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index 708cce0ff..1d156d19c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -94,7 +94,7 @@ namespace net_utils return true; } - inline + inline bool parse_uri(const std::string uri, http::uri_content& content) { @@ -128,11 +128,51 @@ namespace net_utils return true; } + inline + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 - inline + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + inline bool parse_url(const std::string url_str, http::url_content& content) { + if (parse_url_ipv6(url_str, content)) return true; + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index fce01311c..dd80fae8b 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -31,11 +31,13 @@ #include <boost/uuid/uuid.hpp> #include <boost/asio/io_service.hpp> +#include <boost/asio/ip/address_v6.hpp> #include <typeinfo> #include <type_traits> +#include "byte_slice.h" #include "enums.h" -#include "serialization/keyvalue_serialization.h" #include "misc_log_ex.h" +#include "serialization/keyvalue_serialization.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -154,6 +156,59 @@ namespace net_utils inline bool operator>=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept { return !lhs.less(rhs); } + class ipv6_network_address + { + protected: + boost::asio::ip::address_v6 m_address; + uint16_t m_port; + + public: + ipv6_network_address() + : ipv6_network_address(boost::asio::ip::address_v6::loopback(), 0) + {} + + ipv6_network_address(const boost::asio::ip::address_v6& ip, uint16_t port) + : m_address(ip), m_port(port) + { + } + + bool equal(const ipv6_network_address& other) const noexcept; + bool less(const ipv6_network_address& other) const noexcept; + bool is_same_host(const ipv6_network_address& other) const noexcept + { return m_address == other.m_address; } + + boost::asio::ip::address_v6 ip() const noexcept { return m_address; } + uint16_t port() const noexcept { return m_port; } + std::string str() const; + std::string host_str() const; + bool is_loopback() const; + bool is_local() const; + static constexpr address_type get_type_id() noexcept { return address_type::ipv6; } + static constexpr zone get_zone() noexcept { return zone::public_; } + static constexpr bool is_blockable() noexcept { return true; } + + static const uint8_t ID = 2; + BEGIN_KV_SERIALIZE_MAP() + boost::asio::ip::address_v6::bytes_type bytes = this_ref.m_address.to_bytes(); + epee::serialization::selector<is_store>::serialize_t_val_as_blob(bytes, stg, hparent_section, "addr"); + const_cast<boost::asio::ip::address_v6&>(this_ref.m_address) = boost::asio::ip::address_v6(bytes); + KV_SERIALIZE(m_port) + END_KV_SERIALIZE_MAP() + }; + + inline bool operator==(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return lhs.equal(rhs); } + inline bool operator!=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !lhs.equal(rhs); } + inline bool operator<(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return lhs.less(rhs); } + inline bool operator<=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !rhs.less(lhs); } + inline bool operator>(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return rhs.less(lhs); } + inline bool operator>=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !lhs.less(rhs); } + class network_address { struct interface @@ -261,6 +316,8 @@ namespace net_utils { case address_type::ipv4: return this_ref.template serialize_addr<ipv4_network_address>(is_store_, stg, hparent_section); + case address_type::ipv6: + return this_ref.template serialize_addr<ipv6_network_address>(is_store_, stg, hparent_section); case address_type::tor: return this_ref.template serialize_addr<net::tor_address>(is_store_, stg, hparent_section); case address_type::i2p: @@ -368,7 +425,7 @@ namespace net_utils /************************************************************************/ struct i_service_endpoint { - virtual bool do_send(const void* ptr, size_t cb)=0; + virtual bool do_send(byte_slice message)=0; virtual bool close()=0; virtual bool send_done()=0; virtual bool call_run_once_service_io()=0; diff --git a/contrib/epee/include/readline_buffer.h b/contrib/epee/include/readline_buffer.h index 5968d243d..e8f75a9e1 100644 --- a/contrib/epee/include/readline_buffer.h +++ b/contrib/epee/include/readline_buffer.h @@ -40,5 +40,7 @@ namespace rdln readline_buffer* m_buffer; bool m_restart; }; + + void clear_screen(); } diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index b5c4138c5..fe53628a5 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -31,6 +31,9 @@ #include <algorithm> #include <boost/utility/string_ref.hpp> +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + namespace epee { namespace misc_utils @@ -62,6 +65,26 @@ namespace misc_utils 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; + static const constexpr unsigned char isx[256] = + { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; + inline bool isspace(char c) { return lut[(uint8_t)c] & 8; @@ -162,6 +185,42 @@ namespace misc_utils val.push_back('\\');break; case '/': //Slash character val.push_back('/');break; + case 'u': //Unicode code point + if (buf_end - it < 4) + { + ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); + } + else + { + uint32_t dst = 0; + for (int i = 0; i < 4; ++i) + { + const unsigned char tmp = isx[(int)*++it]; + CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); + dst = dst << 4 | tmp; + } + // encode as UTF-8 + if (dst <= 0x7f) + { + val.push_back(dst); + } + else if (dst <= 0x7ff) + { + val.push_back(0xc0 | (dst >> 6)); + val.push_back(0x80 | (dst & 0x3f)); + } + else if (dst <= 0xffff) + { + val.push_back(0xe0 | (dst >> 12)); + val.push_back(0x80 | ((dst >> 6) & 0x3f)); + val.push_back(0x80 | (dst & 0x3f)); + } + else + { + ASSERT_MES_AND_THROW("Unicode code point is out or range"); + } + } + break; default: val.push_back(*it); LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index da47b7d55..1be5eb5e1 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -59,26 +59,6 @@ #pragma comment (lib, "Rpcrt4.lib") #endif -static const constexpr unsigned char isx[256] = -{ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -}; - namespace epee { namespace string_tools @@ -99,10 +79,10 @@ namespace string_tools for(size_t i = 0; i < s.size(); i += 2) { int tmp = *src++; - tmp = isx[tmp]; + tmp = epee::misc_utils::parse::isx[tmp]; if (tmp == 0xff) return false; int t2 = *src++; - t2 = isx[t2]; + t2 = epee::misc_utils::parse::isx[t2]; if (t2 == 0xff) return false; *dst++ = (tmp << 4) | t2; } diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index 9f2404856..dba02f270 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -150,81 +150,6 @@ namespace epee }; -#if defined(WINDWOS_PLATFORM) - class shared_critical_section - { - public: - shared_critical_section() - { - ::InitializeSRWLock(&m_srw_lock); - } - ~shared_critical_section() - {} - - bool lock_shared() - { - AcquireSRWLockShared(&m_srw_lock); - return true; - } - bool unlock_shared() - { - ReleaseSRWLockShared(&m_srw_lock); - return true; - } - bool lock_exclusive() - { - ::AcquireSRWLockExclusive(&m_srw_lock); - return true; - } - bool unlock_exclusive() - { - ::ReleaseSRWLockExclusive(&m_srw_lock); - return true; - } - private: - SRWLOCK m_srw_lock; - }; - - - class shared_guard - { - public: - shared_guard(shared_critical_section& ref_sec):m_ref_sec(ref_sec) - { - m_ref_sec.lock_shared(); - } - - ~shared_guard() - { - m_ref_sec.unlock_shared(); - } - - private: - shared_critical_section& m_ref_sec; - }; - - - class exclusive_guard - { - public: - exclusive_guard(shared_critical_section& ref_sec):m_ref_sec(ref_sec) - { - m_ref_sec.lock_exclusive(); - } - - ~exclusive_guard() - { - m_ref_sec.unlock_exclusive(); - } - - private: - shared_critical_section& m_ref_sec; - }; -#endif - -#define SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard critical_region_var(x) -#define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard critical_region_var(x) - #define CRITICAL_REGION_LOCAL(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var(x) #define CRITICAL_REGION_BEGIN(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)> critical_region_var(x) #define CRITICAL_REGION_LOCAL1(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var1(x) @@ -232,22 +157,6 @@ namespace epee #define CRITICAL_REGION_END() } - -#if defined(WINDWOS_PLATFORM) - inline const char* get_wait_for_result_as_text(DWORD res) - { - switch(res) - { - case WAIT_ABANDONED: return "WAIT_ABANDONED"; - case WAIT_TIMEOUT: return "WAIT_TIMEOUT"; - case WAIT_OBJECT_0: return "WAIT_OBJECT_0"; - case WAIT_OBJECT_0+1: return "WAIT_OBJECT_1"; - case WAIT_OBJECT_0+2: return "WAIT_OBJECT_2"; - default: return "UNKNOWN CODE"; - } - } -#endif - } #endif diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 2465afebb..c512e3b86 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,8 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c - connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp) +add_library(epee STATIC byte_slice.cpp hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp + levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp) if (USE_READLINE AND GNU_READLINE_FOUND) add_library(epee_readline STATIC readline_buffer.cpp) diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp new file mode 100644 index 000000000..216049e5b --- /dev/null +++ b/contrib/epee/src/byte_slice.cpp @@ -0,0 +1,209 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <atomic> +#include <cstdlib> +#include <cstring> +#include <limits> +#include <stdexcept> +#include <utility> + +#include "byte_slice.h" + +namespace epee +{ + struct byte_slice_data + { + byte_slice_data() noexcept + : ref_count(1) + {} + + virtual ~byte_slice_data() noexcept + {} + + std::atomic<std::size_t> ref_count; + }; + + void release_byte_slice::operator()(byte_slice_data* ptr) const noexcept + { + if (ptr && --(ptr->ref_count) == 0) + { + ptr->~byte_slice_data(); + free(ptr); + } + } + + namespace + { + template<typename T> + struct adapted_byte_slice final : byte_slice_data + { + explicit adapted_byte_slice(T&& buffer) + : byte_slice_data(), buffer(std::move(buffer)) + {} + + virtual ~adapted_byte_slice() noexcept final override + {} + + const T buffer; + }; + + // bytes "follow" this structure in memory slab + struct raw_byte_slice final : byte_slice_data + { + raw_byte_slice() noexcept + : byte_slice_data() + {} + + virtual ~raw_byte_slice() noexcept final override + {} + }; + + /* This technique is not-standard, but allows for the reference count and + memory for the bytes (when given a list of spans) to be allocated in a + single call. In that situation, the dynamic sized bytes are after/behind + the raw_byte_slice class. The C runtime has to track the number of bytes + allocated regardless, so free'ing is relatively easy. */ + + template<typename T, typename... U> + std::unique_ptr<T, release_byte_slice> allocate_slice(std::size_t extra_bytes, U&&... args) + { + if (std::numeric_limits<std::size_t>::max() - sizeof(T) < extra_bytes) + throw std::bad_alloc{}; + + void* const ptr = malloc(sizeof(T) + extra_bytes); + if (ptr == nullptr) + throw std::bad_alloc{}; + + try + { + new (ptr) T{std::forward<U>(args)...}; + } + catch (...) + { + free(ptr); + throw; + } + return std::unique_ptr<T, release_byte_slice>{reinterpret_cast<T*>(ptr)}; + } + } // anonymous + + byte_slice::byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept + : storage_(storage), portion_(portion) + { + if (storage_) + ++(storage_->ref_count); + } + + template<typename T> + byte_slice::byte_slice(const adapt_buffer, T&& buffer) + : storage_(nullptr), portion_(to_byte_span(to_span(buffer))) + { + if (!buffer.empty()) + storage_ = allocate_slice<adapted_byte_slice<T>>(0, std::move(buffer)); + } + + byte_slice::byte_slice(std::initializer_list<span<const std::uint8_t>> sources) + : byte_slice() + { + std::size_t space_needed = 0; + for (const auto source : sources) + space_needed += source.size(); + + if (space_needed) + { + auto storage = allocate_slice<raw_byte_slice>(space_needed); + span<std::uint8_t> out{reinterpret_cast<std::uint8_t*>(storage.get() + 1), space_needed}; + portion_ = {out.data(), out.size()}; + + for (const auto source : sources) + { + std::memcpy(out.data(), source.data(), source.size()); + if (out.remove_prefix(source.size()) < source.size()) + throw std::bad_alloc{}; // size_t overflow on space_needed + } + storage_ = std::move(storage); + } + } + + byte_slice::byte_slice(std::string&& buffer) + : byte_slice(adapt_buffer{}, std::move(buffer)) + {} + + byte_slice::byte_slice(std::vector<std::uint8_t>&& buffer) + : byte_slice(adapt_buffer{}, std::move(buffer)) + {} + + byte_slice::byte_slice(byte_slice&& source) noexcept + : storage_(std::move(source.storage_)), portion_(source.portion_) + { + source.portion_ = epee::span<const std::uint8_t>{}; + } + + byte_slice& byte_slice::operator=(byte_slice&& source) noexcept + { + storage_ = std::move(source.storage_); + portion_ = source.portion_; + if (source.storage_ == nullptr) + source.portion_ = epee::span<const std::uint8_t>{}; + + return *this; + } + + std::size_t byte_slice::remove_prefix(std::size_t max_bytes) noexcept + { + max_bytes = portion_.remove_prefix(max_bytes); + if (portion_.empty()) + storage_ = nullptr; + return max_bytes; + } + + byte_slice byte_slice::take_slice(const std::size_t max_bytes) noexcept + { + byte_slice out{}; + std::uint8_t const* const ptr = data(); + out.portion_ = {ptr, portion_.remove_prefix(max_bytes)}; + + if (portion_.empty()) + out.storage_ = std::move(storage_); // no atomic inc/dec + else + out = {storage_.get(), out.portion_}; + + return out; + } + + byte_slice byte_slice::get_slice(const std::size_t begin, const std::size_t end) const + { + if (end < begin || portion_.size() < end) + throw std::out_of_range{"bad slice range"}; + + if (begin == end) + return {}; + return {storage_.get(), {portion_.begin() + begin, end - begin}}; + } +} // epee diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index 82d9e3b53..7526dde26 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -128,7 +128,7 @@ connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_thro int connection_basic_pimpl::m_default_tos; // methods: -connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) +connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) : m_state(std::move(state)), mI( new connection_basic_pimpl("peer") ), @@ -152,7 +152,7 @@ connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, boost::s _note("Spawned connection #"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count); } -connection_basic::connection_basic(boost::asio::io_service &io_service, boost::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) +connection_basic::connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support) : m_state(std::move(state)), mI( new connection_basic_pimpl("peer") ), diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp new file mode 100644 index 000000000..ff845e2a7 --- /dev/null +++ b/contrib/epee/src/levin_base.cpp @@ -0,0 +1,128 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/levin_base.h" + +#include "int-util.h" + +namespace epee +{ +namespace levin +{ + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept + { + bucket_head2 head = {0}; + head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + head.m_have_to_return_data = expect_response; + head.m_cb = SWAP64LE(msg_size); + + head.m_command = SWAP32LE(command); + head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); + head.m_flags = SWAP32LE(flags); + return head; + } + + byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) + { + const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); + return byte_slice{epee::as_byte_span(head), payload}; + } + + byte_slice make_noise_notify(const std::size_t noise_bytes) + { + static constexpr const std::uint32_t flags = + LEVIN_PACKET_BEGIN | LEVIN_PACKET_END; + + if (noise_bytes < sizeof(bucket_head2)) + return nullptr; + + std::string buffer(noise_bytes, char(0)); + const bucket_head2 head = make_header(0, noise_bytes - sizeof(bucket_head2), flags, false); + std::memcpy(std::addressof(buffer[0]), std::addressof(head), sizeof(head)); + + return byte_slice{std::move(buffer)}; + } + + byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + { + const size_t noise_size = noise_message.size(); + if (noise_size < sizeof(bucket_head2) * 2) + return nullptr; + + if (payload.size() <= noise_size - sizeof(bucket_head2)) + { + /* The entire message can be sent at once, and the levin binary parser + will ignore extra bytes. So just pad with zeroes and otherwise send + a "normal", not fragmented message. */ + const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); + const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; + + const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); + return byte_slice{as_byte_span(head), payload, padding_bytes}; + } + + // fragment message + const size_t payload_space = noise_size - sizeof(bucket_head2); + const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; + + std::string buffer{}; + buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + + bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); + buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + + head.m_command = command; + head.m_flags = LEVIN_PACKET_REQUEST; + head.m_cb = payload.size(); + buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + + size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); + buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + + head.m_command = 0; + head.m_flags = 0; + head.m_cb = noise_size - sizeof(bucket_head2); + + while (!payload.empty()) + { + copy_size = payload.remove_prefix(payload_space); + + if (payload.empty()) + head.m_flags = LEVIN_PACKET_END; + + buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + } + + const size_t padding = noise_size - copy_size - sizeof(bucket_head2); + buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + + return byte_slice{std::move(buffer)}; + } +} // levin +} // epee diff --git a/contrib/epee/src/net_helper.cpp b/contrib/epee/src/net_helper.cpp index 3543f5716..719f1c8e0 100644 --- a/contrib/epee/src/net_helper.cpp +++ b/contrib/epee/src/net_helper.cpp @@ -11,10 +11,39 @@ namespace net_utils ////////////////////////////////////////////////////////////////////////// boost::asio::ip::tcp::resolver resolver(GET_IO_SERVICE(timeout)); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver::iterator iterator; boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty - throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + boost::system::error_code resolve_error; + try + { + iterator = resolver.resolve(query, resolve_error); + if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty + { + // if IPv4 resolution fails, try IPv6. Unintentional outgoing IPv6 connections should only + // be possible if for some reason a hostname was given and that hostname fails IPv4 resolution, + // so at least for now there should not be a need for a flag "using ipv6 is ok" + try_ipv6 = true; + } + + } + catch (const boost::system::system_error& e) + { + if (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again) + { + throw; + } + try_ipv6 = true; + } + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + iterator = resolver.resolve(query6); + if (iterator == end) + throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + } ////////////////////////////////////////////////////////////////////////// diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index b7f07a23b..5cc49cc71 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -21,6 +21,19 @@ namespace epee { namespace net_utils bool ipv4_network_address::is_loopback() const { return net_utils::is_ip_loopback(ip()); } bool ipv4_network_address::is_local() const { return net_utils::is_ip_local(ip()); } + bool ipv6_network_address::equal(const ipv6_network_address& other) const noexcept + { return is_same_host(other) && port() == other.port(); } + + bool ipv6_network_address::less(const ipv6_network_address& other) const noexcept + { return is_same_host(other) ? port() < other.port() : m_address < other.m_address; } + + std::string ipv6_network_address::str() const + { return std::string("[") + host_str() + "]:" + std::to_string(port()); } + + std::string ipv6_network_address::host_str() const { return m_address.to_string(); } + bool ipv6_network_address::is_loopback() const { return m_address.is_loopback(); } + bool ipv6_network_address::is_local() const { return m_address.is_link_local(); } + bool ipv4_network_subnet::equal(const ipv4_network_subnet& other) const noexcept { return is_same_host(other) && m_mask == other.m_mask; } diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index 39369c43f..05322b693 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -71,6 +71,11 @@ rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const { boost::lock_guard<boost::mutex> lock(sync_mutex); line_stat = rdln::partial; + if (!m_cout_buf) + { + line = ""; + return rdln::full; + } rl_callback_read_char(); if (line_stat == rdln::full) { @@ -224,3 +229,8 @@ static void remove_line_handler() rl_callback_handler_remove(); } +void rdln::clear_screen() +{ + rl_clear_screen(0, 0); +} + diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index 3fc26354c..32aee5f56 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -26,12 +26,16 @@ Preparing the Gitian builder host The first step is to prepare the host environment that will be used to perform the Gitian builds. This guide explains how to set up the environment, and how to start the builds. -Gitian offers to build with either `kvm`, `docker` or `lxc`. The default build -path chosen is `lxc`, but its setup is more complicated. You need to be logged in as the `gitianuser`. -If this user does not exist yet on your system, create it. Gitian can use -either kvm, lxc or docker as a host environment. This documentation will show -how to build with lxc and docker. While the docker setup is easy, the lxc setup -is more involved. + +* Gitian host OS should be Ubuntu 18.04 "Bionic Beaver". If you are on a mac or windows for example, you can run it in a VM but will be slower. + +* Gitian gives you the option of using any of 3 different virtualization tools: `kvm`, `docker` or `lxc`. This documentation will only show how to build with `lxc` and `docker` (documentation for `kvm` is welcome). Building with `lxc` is the default, but is more complicated, so we recommend docker your first time. + + +## Create the gitianuser account + +You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. + LXC --- @@ -76,10 +80,17 @@ This setup is required to enable networking in the container. Docker ------ -Building in docker does not require much setup. Install docker on your host, then type the following: +Prepare for building with docker: ```bash -sudo apt-get install git make curl +sudo apt-get install git make curl docker.io +``` + +Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): + +```bash +sudo groupadd docker +sudo usermod -aG docker gitianuser ``` Optionally add yourself to the docker group. Note that this will give docker root access to your system. @@ -88,7 +99,7 @@ Optionally add yourself to the docker group. Note that this will give docker roo sudo usermod -aG docker gitianuser ``` -Manual and Building +Manual Building ------------------- The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. @@ -109,66 +120,95 @@ The `gitian-build.py` script will checkout different release tags, so it's best cp monero/contrib/gitian/gitian-build.py . ``` -Setup the required environment, you only need to do this once: +### Setup the required environment + +Setup for LXC: ```bash -./gitian-build.py --setup fluffypony v0.14.0 +GH_USER=fluffypony +VERSION=v0.14.1.0 + +./gitian-build.py --setup $GH_USER $VERSION ``` -Where `fluffypony` is your Github name and `v0.14.0` is the version tag you want to build. -If you are using docker, run it with: +Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. + +Setup for docker: ```bash -./gitian-build.py --setup --docker fluffypony v0.14.0 +./gitian-build.py --setup --docker $GH_USER $VERSION ``` -While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. -This script is only there for convenience. Seperate steps for building can still be taken. +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a separate step. This script is only there for convenience. Separate steps for building can still be taken. In order to sign gitian builds on your host machine, which has your PGP key, -fork the gitian.sigs repository and clone it on your host machine, +fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) and clone it on your host machine, or pass the signed assert file back to your build machine. ```bash git clone git@github.com:monero-project/gitian.sigs.git -git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git +git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git ``` -Build Binaries ------------------------------ -To build the most recent tag (pass in `--docker` after setting up with docker): +Build the binaries +------------------ + +**Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. + +To build the most recent tag (pass in `--docker` if using docker): ```bash -./gitian-build.py --detach-sign --no-commit -b fluffypony v0.14.0 +./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION ``` -To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. +To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. -If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. +If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. -If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: +Checking your work +------------------ + +Take a look in the assert files and note the SHA256 checksums listed there. eg for `v0.14.1.0` you should get this checksum: + +``` +2b95118f53d98d542a85f8732b84ba13b3cd20517ccb40332b0edd0ddf4f8c62 monero-x86_64-linux-gnu.tar.gz +``` + +You should verify that this is really the checksum you get on that file you built. You can also look in the gitian.sigs repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. + + +Signing assert files +-------------------- + +If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -export NAME=fluffypony -export VERSION=v0.14.0 -gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert -gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert -gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert +GH_USER=fluffypony +VERSION=v0.14.1.0 + +gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert +gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert +gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert ``` +<!-- TODO: Replace * above with ${VERSION} once gitian builds correct file name --> + +This will create a `.sig` file for each `.assert` file above (2 files for each platform). + + +Submitting your signed assert files +----------------------------------- Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash -git checkout -b v0.14.0 -git commit -S -a -m "Add $NAME v0.14.0" -git push --set-upstream $NAME v0.14.0 +git checkout -b $VERSION +# add your assert and sig files... +git commit -S -a -m "Add $GH_USER $VERSION" +git push --set-upstream $GH_USER $VERSION ``` -```bash -gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert -gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert -gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert -``` +**Note:** Please ensure your gpg public key is available to check signatures by adding it to the [gitian.sigs/gitian-pubkeys/](https://github.com/monero-project/gitian.sigs/tree/master/gitian-pubkeys) directory in a pull request. + More Build Options ------------------ diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index b654b15c7..5913fda3a 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -54,7 +54,7 @@ def build(): print('\nCompiling ' + args.version + ' Linux') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) - subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero-*.tar.bz2 ../monero-binaries/'+args.version, shell=True) if args.windows: print('\nCompiling ' + args.version + ' Windows') @@ -64,9 +64,9 @@ def build(): if args.macos: print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero*.tar.bz2 ../monero-binaries/'+args.version, shell=True) os.chdir(workdir) @@ -141,8 +141,8 @@ def main(): if not 'LXC_GUEST_IP' in os.environ.keys(): os.environ['LXC_GUEST_IP'] = '10.0.3.5' - # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + # Disable MacOS build if no SDK found + if args.build and args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 6e3ad108e..8cbde2acb 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -131,6 +131,13 @@ script: | git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host for i in $HOSTS; do @@ -155,9 +162,10 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON make ${MAKEOPTS} - DISTNAME=monero-${i} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 cd .. rm -rf build done + diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 1cdb85273..d3141e2c7 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -79,6 +79,13 @@ script: | git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends mkdir -p ${BASEPREFIX}/SDKs @@ -102,9 +109,9 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make ${MAKEOPTS} - DISTNAME=monero-${i} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 cd .. rm -rf build done diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index d15383905..4c559acfe 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -102,6 +102,13 @@ script: | git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host for i in $HOSTS; do @@ -127,7 +134,7 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make ${MAKEOPTS} - DISTNAME=monero-${i} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip cd .. && rm -rf build |