diff options
author | Riccardo Spagni <ric@spagni.net> | 2017-03-06 15:17:08 +0000 |
---|---|---|
committer | Riccardo Spagni <ric@spagni.net> | 2017-03-06 15:17:08 +0000 |
commit | ba0767477d58033824fb3a48134a0b4c14100d24 (patch) | |
tree | e09471953b4cd85af8b04428525bd4d6a9400219 | |
parent | Merge pull request #1848 (diff) | |
parent | rpc: fix BUILD_TAG mispelling (BUILDTAG) (diff) | |
download | monero-ba0767477d58033824fb3a48134a0b4c14100d24.tar.xz |
Merge pull request #1841
b553c282 rpc: fix BUILD_TAG mispelling (BUILDTAG) (moneromooo-monero)
02097c87 core: print the "new update found" message in cyan, for visibility (moneromooo-monero)
749ebace download: check available disk space before downloading (moneromooo-monero)
f36c5f1e download: give download threads distinct names (moneromooo-monero)
f6211322 core: make update download cancellable (moneromooo-monero)
63f0e074 download: async API (moneromooo-monero)
9bf017ed http_client: allow cancelling a download (moneromooo-monero)
0d90123c http_client: allow derived class to get headers at start (moneromooo-monero)
-rw-r--r-- | contrib/epee/include/net/http_client.h | 23 | ||||
-rw-r--r-- | src/common/download.cpp | 182 | ||||
-rw-r--r-- | src/common/download.h | 10 | ||||
-rw-r--r-- | src/cryptonote_core/cryptonote_core.cpp | 69 | ||||
-rw-r--r-- | src/cryptonote_core/cryptonote_core.h | 5 | ||||
-rw-r--r-- | src/rpc/core_rpc_server.cpp | 4 |
6 files changed, 245 insertions, 48 deletions
diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index d73eda39c..67e63f7bf 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -337,6 +337,11 @@ using namespace std; return true; } //--------------------------------------------------------------------------- + virtual bool on_header(const http_response_info &headers) + { + return true; + } + //--------------------------------------------------------------------------- inline bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) { @@ -505,6 +510,12 @@ using namespace std; m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end()); analize_cached_header_and_invoke_state(); + if (!on_header(m_response_info)) + { + MDEBUG("Connection cancelled by on_header"); + m_state = reciev_machine_state_done; + return false; + } m_header_cache.clear(); if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done)) need_more_data = true; @@ -527,7 +538,11 @@ using namespace std; } CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()"); m_len_in_remain -= recv_buff.size(); - m_pcontent_encoding_handler->update_in(recv_buff); + if (!m_pcontent_encoding_handler->update_in(recv_buff)) + { + m_state = reciev_machine_state_done; + return false; + } if(m_len_in_remain == 0) m_state = reciev_machine_state_done; @@ -700,7 +715,11 @@ using namespace std; m_len_in_remain = 0; } - m_pcontent_encoding_handler->update_in(chunk_body); + if (!m_pcontent_encoding_handler->update_in(chunk_body)) + { + m_state = reciev_machine_state_error; + return false; + } if(!m_len_in_remain) m_chunked_state = http_chunked_state_chunk_head; diff --git a/src/common/download.cpp b/src/common/download.cpp index c5ee797d0..28aac5a59 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <string> +#include <atomic> #include <boost/filesystem.hpp> #include <boost/asio.hpp> #include <boost/thread/thread.hpp> @@ -40,27 +41,82 @@ namespace tools { - static bool download_thread(const std::string &path, const std::string &url) + struct download_thread_control { + const std::string path; + const std::string uri; + std::function<void(const std::string&, const std::string&, bool)> result_cb; + std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb; + bool stop; + bool stopped; + bool success; + boost::thread thread; + boost::mutex mutex; + + download_thread_control(const std::string &path, const std::string &uri, std::function<void(const std::string&, const std::string&, bool)> result_cb, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb): + path(path), uri(uri), result_cb(result_cb), progress_cb(progress_cb), stop(false), stopped(false), success(false) {} + ~download_thread_control() { if (thread.joinable()) thread.detach(); } + }; + + static void download_thread(download_async_handle control) + { + static std::atomic<unsigned int> thread_id(0); + + MLOG_SET_THREAD_NAME("DL" + std::to_string(thread_id++)); + + struct stopped_setter + { + stopped_setter(const download_async_handle &control): control(control) {} + ~stopped_setter() { control->stopped = true; } + download_async_handle control; + } stopped_setter(control); + try { - MINFO("Downloading " << url << " to " << path); + boost::unique_lock<boost::mutex> lock(control->mutex); + MINFO("Downloading " << control->uri << " to " << control->path); std::ofstream f; - f.open(path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + f.open(control->path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); if (!f.good()) { - MERROR("Failed to open file " << path); - return false; + MERROR("Failed to open file " << control->path); + control->result_cb(control->path, control->uri, control->success); + return; } class download_client: public epee::net_utils::http::http_simple_client { public: - download_client(std::ofstream &f): f(f) {} + download_client(download_async_handle control, std::ofstream &f): + control(control), f(f), content_length(-1), total(0) {} virtual ~download_client() { f.close(); } + virtual bool on_header(const epee::net_utils::http::http_response_info &headers) + { + ssize_t length; + if (epee::string_tools::get_xtype_from_string(length, headers.m_header_info.m_content_length) && length >= 0) + { + MINFO("Content-Length: " << length); + content_length = length; + boost::filesystem::path path(control->path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if (si.available < (size_t)content_length) + { + const uint64_t avail = (si.available + 1023) / 1024, needed = (content_length + 1023) / 1024; + MERROR("Not enough space to download " << needed << " kB to " << path << " (" << avail << " kB available)"); + return false; + } + } + return true; + } virtual bool handle_target_data(std::string &piece_of_transfer) { try { + boost::lock_guard<boost::mutex> lock(control->mutex); + if (control->stop) + return false; f << piece_of_transfer; + total += piece_of_transfer.size(); + if (control->progress_cb && !control->progress_cb(control->path, control->uri, total, content_length)) + return false; return f.good(); } catch (const std::exception &e) @@ -70,69 +126,145 @@ namespace tools } } private: + download_async_handle control; std::ofstream &f; - } client(f); + ssize_t content_length; + size_t total; + } client(control, f); epee::net_utils::http::url_content u_c; - if (!epee::net_utils::parse_url(url, u_c)) + if (!epee::net_utils::parse_url(control->uri, u_c)) { - MWARNING("Failed to parse URL " << url); - return false; + MERROR("Failed to parse URL " << control->uri); + control->result_cb(control->path, control->uri, control->success); + return; } if (u_c.host.empty()) { - MWARNING("Failed to determine address from URL " << url); - return false; + MERROR("Failed to determine address from URL " << control->uri); + control->result_cb(control->path, control->uri, control->success); + return; } + + lock.unlock(); + uint16_t port = u_c.port ? u_c.port : 80; MDEBUG("Connecting to " << u_c.host << ":" << port); client.set_server(u_c.host, std::to_string(port), boost::none); if (!client.connect(std::chrono::seconds(30))) { - MERROR("Failed to connect to " << url); - return false; + boost::lock_guard<boost::mutex> lock(control->mutex); + MERROR("Failed to connect to " << control->uri); + control->result_cb(control->path, control->uri, control->success); + return; } MDEBUG("GETting " << u_c.uri); const epee::net_utils::http::http_response_info *info = NULL; if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info)) { - MERROR("Failed to connect to " << url); + boost::lock_guard<boost::mutex> lock(control->mutex); + MERROR("Failed to connect to " << control->uri); client.disconnect(); - return false; + control->result_cb(control->path, control->uri, control->success); + return; + } + if (control->stop) + { + boost::lock_guard<boost::mutex> lock(control->mutex); + MDEBUG("Download cancelled"); + client.disconnect(); + control->result_cb(control->path, control->uri, control->success); + return; } if (!info) { - MERROR("Failed invoking GET command to " << url << ", no status info returned"); + boost::lock_guard<boost::mutex> lock(control->mutex); + MERROR("Failed invoking GET command to " << control->uri << ", no status info returned"); client.disconnect(); - return false; + control->result_cb(control->path, control->uri, control->success); + return; } MDEBUG("response code: " << info->m_response_code); + MDEBUG("response length: " << info->m_header_info.m_content_length); MDEBUG("response comment: " << info->m_response_comment); MDEBUG("response body: " << info->m_body); for (const auto &f: info->m_additional_fields) MDEBUG("additional field: " << f.first << ": " << f.second); if (info->m_response_code != 200) { + boost::lock_guard<boost::mutex> lock(control->mutex); MERROR("Status code " << info->m_response_code); client.disconnect(); - return false; + control->result_cb(control->path, control->uri, control->success); + return; } client.disconnect(); f.close(); MDEBUG("Download complete"); - return true; + lock.lock(); + control->success = true; + control->result_cb(control->path, control->uri, control->success); + return; } catch (const std::exception &e) { MERROR("Exception in download thread: " << e.what()); - return false; + // fall through and call result_cb not from the catch block to avoid another exception } + boost::lock_guard<boost::mutex> lock(control->mutex); + control->result_cb(control->path, control->uri, control->success); } - bool download(const std::string &path, const std::string &url) + bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> cb) { - bool success; - std::unique_ptr<boost::thread> thread(new boost::thread([&](){ success = download_thread(path, url); })); - thread->join(); + bool success = false; + download_async_handle handle = download_async(path, url, [&success](const std::string&, const std::string&, bool result) {success = result;}, cb); + download_wait(handle); return success; } + + download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress) + { + download_async_handle control = std::make_shared<download_thread_control>(path, url, result, progress); + control->thread = boost::thread([control](){ download_thread(control); }); + return control; + } + + bool download_finished(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + boost::lock_guard<boost::mutex> lock(control->mutex); + return control->stopped; + } + + bool download_error(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + boost::lock_guard<boost::mutex> lock(control->mutex); + return !control->success; + } + + bool download_wait(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + { + boost::lock_guard<boost::mutex> lock(control->mutex); + if (control->stopped) + return true; + } + control->thread.join(); + return true; + } + + bool download_cancel(const download_async_handle &control) + { + CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle"); + { + boost::lock_guard<boost::mutex> lock(control->mutex); + if (control->stopped) + return true; + control->stop = true; + } + control->thread.join(); + return true; + } } diff --git a/src/common/download.h b/src/common/download.h index ab7644689..917cb2278 100644 --- a/src/common/download.h +++ b/src/common/download.h @@ -32,5 +32,13 @@ namespace tools { - bool download(const std::string &path, const std::string &url); + struct download_thread_control; + typedef std::shared_ptr<download_thread_control> download_async_handle; + + bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL); + download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL); + bool download_error(const download_async_handle &h); + bool download_finished(const download_async_handle &h); + bool download_wait(const download_async_handle &h); + bool download_cancel(const download_async_handle &h); } diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index cfe3b5441..9c5fae4ef 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -75,7 +75,8 @@ namespace cryptonote m_target_blockchain_height(0), m_checkpoints_path(""), m_last_dns_checkpoints_update(0), - m_last_json_checkpoints_update(0) + m_last_json_checkpoints_update(0), + m_update_download(0) { set_cryptonote_protocol(pprotocol); } @@ -134,6 +135,15 @@ namespace cryptonote void core::stop() { m_blockchain_storage.cancel(); + + tools::download_async_handle handle; + { + boost::lock_guard<boost::mutex> lock(m_update_mutex); + handle = m_update_download; + m_update_download = 0; + } + if (handle) + tools::download_cancel(handle); } //----------------------------------------------------------------------------------- void core::init_options(boost::program_options::options_description& desc) @@ -1118,7 +1128,7 @@ namespace cryptonote return true; std::string url = tools::get_update_url(software, subdir, buildtag, version, true); - MGINFO("Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); + MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); if (check_updates_level == UPDATES_NOTIFY) return true; @@ -1133,32 +1143,55 @@ namespace cryptonote boost::filesystem::path path(epee::string_tools::get_current_module_folder()); path /= filename; + boost::unique_lock<boost::mutex> lock(m_update_mutex); + + if (m_update_download != 0) + { + MCDEBUG("updates", "Already downloading update"); + return true; + } + crypto::hash file_hash; if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash))) { MCDEBUG("updates", "We don't have that file already, downloading"); - if (!tools::download(path.string(), url)) - { - MCERROR("updates", "Failed to download " << url); - return false; - } - if (!tools::sha256sum(path.string(), file_hash)) - { - MCERROR("updates", "Failed to hash " << path); - return false; - } - if (hash != epee::string_tools::pod_to_hex(file_hash)) - { - MCERROR("updates", "Download from " << url << " does not match the expected hash"); - return false; - } - MGINFO("New version downloaded to " << path); + m_last_update_length = 0; + m_update_download = tools::download_async(path.string(), url, [this, hash](const std::string &path, const std::string &uri, bool success) { + if (success) + { + crypto::hash file_hash; + if (!tools::sha256sum(path, file_hash)) + { + MCERROR("updates", "Failed to hash " << path); + } + if (hash != epee::string_tools::pod_to_hex(file_hash)) + { + MCERROR("updates", "Download from " << uri << " does not match the expected hash"); + } + MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path); + } + else + { + MCERROR("updates", "Failed to download " << uri); + } + boost::unique_lock<boost::mutex> lock(m_update_mutex); + m_update_download = 0; + }, [this](const std::string &path, const std::string &uri, size_t length, ssize_t content_length) { + if (length >= m_last_update_length + 1024 * 1024 * 10) + { + m_last_update_length = length; + MCDEBUG("updates", "Downloaded " << length << "/" << (content_length ? std::to_string(content_length) : "unknown")); + } + return true; + }); } else { MCDEBUG("updates", "We already have " << path << " with expected hash"); } + lock.unlock(); + if (check_updates_level == UPDATES_DOWNLOAD) return true; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 7323bef29..02d58691d 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,6 +39,7 @@ #include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" +#include "common/download.h" #include "tx_pool.h" #include "blockchain.h" #include "cryptonote_basic/miner.h" @@ -844,6 +845,10 @@ namespace cryptonote UPDATES_DOWNLOAD, UPDATES_UPDATE, } check_updates_level; + + tools::download_async_handle m_update_download; + size_t m_last_update_length; + boost::mutex m_update_mutex; }; } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index b114f77ae..9f7e8aa38 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1491,8 +1491,8 @@ namespace cryptonote bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res) { static const char software[] = "monero"; -#ifdef BUILDTAG - static const char buildtag[] = BOOST_PP_STRINGIZE(BUILDTAG); +#ifdef BUILD_TAG + static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG); #else static const char buildtag[] = "source"; #endif |