diff options
45 files changed, 1096 insertions, 406 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a3dbf974a..311af76dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,8 +176,8 @@ if(NOT MANUAL_SUBMODULES) find_package(Git) if(GIT_FOUND) function (check_submodule relative_path) - execute_process(COMMAND git -C ${relative_path} rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE localHead) - execute_process(COMMAND git rev-parse "HEAD:${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead) + execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE localHead) + execute_process(COMMAND git rev-parse "HEAD:${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead) string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate) if (upToDate) message(STATUS "Submodule '${relative_path}' is up-to-date") @@ -898,9 +898,9 @@ if(MINGW) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wa,-mbig-obj") set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi;crypt32;bcrypt) if(DEPENDS) - set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} sicuio sicuin sicuuc sicudt sicutu iconv) + set(ICU_LIBRARIES sicuio sicuin sicuuc sicudt sicutu iconv) else() - set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} icuio icuin icuuc icudt icutu iconv) + set(ICU_LIBRARIES icuio icuin icuuc icudt icutu iconv) endif() elseif(APPLE OR OPENBSD OR ANDROID) set(EXTRA_LIBRARIES "") diff --git a/Dockerfile b/Dockerfile index cd3e7df70..86d833a98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,9 @@ RUN set -ex && \ libtool-bin \ autoconf \ automake \ - bzip2 + bzip2 \ + xsltproc \ + gperf WORKDIR /usr/local @@ -109,6 +111,42 @@ RUN set -ex \ && make check \ && make install +# Udev +ARG UDEV_VERSION=v3.2.6 +ARG UDEV_HASH=0c35b136c08d64064efa55087c54364608e65ed6 +RUN set -ex \ + && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ + && cd eudev \ + && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ + && ./autogen.sh \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ + && make \ + && make install + +# Libusb +ARG USB_VERSION=v1.0.22 +ARG USB_HASH=0034b2afdcdb1614e78edaa2a9e22d5936aeae5d +RUN set -ex \ + && git clone https://github.com/libusb/libusb.git -b ${USB_VERSION} \ + && cd libusb \ + && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ + && ./autogen.sh \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared \ + && make \ + && make install + +# Hidapi +ARG HIDAPI_VERSION=hidapi-0.8.0-rc1 +ARG HIDAPI_HASH=40cf516139b5b61e30d9403a48db23d8f915f52c +RUN set -ex \ + && git clone https://github.com/signal11/hidapi -b ${HIDAPI_VERSION} \ + && cd hidapi \ + && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ + && ./bootstrap \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && make \ + && make install + WORKDIR /src COPY . . @@ -16,6 +16,15 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - Our [Vulnerability Response Process](https://github.com/monero-project/meta/blob/master/VULNERABILITY_RESPONSE_PROCESS.md) encourages responsible disclosure - We are also available via [HackerOne](https://hackerone.com/monero) +## Research + +The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. + +Our researchers are available on IRC in [#monero-research-lab on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-research-lab&prompt=1&uio=d4) or by email: + +- Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) +- Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) + ## Announcements - You can subscribe to an [announcement listserv](https://lists.getmonero.org) to get critical announcements from the Monero core team. The announcement list can be very helpful for knowing when software updates are needed. @@ -236,7 +245,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( ``` sudo /etc/init.d/dphys-swapfile stop sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=1024 + CONF_SWAPSIZE=2048 sudo /etc/init.d/dphys-swapfile start ``` * If using an external hard disk without an external power supply, ensure it gets enough power to avoid hardware issues when syncing, by adding the line "max_usb_current=1" to /boot/config.txt @@ -270,7 +279,7 @@ If you are using the older Raspbian Jessie image, compiling Monero is a bit more ``` sudo /etc/init.d/dphys-swapfile stop sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=1024 + CONF_SWAPSIZE=2048 sudo /etc/init.d/dphys-swapfile start ``` diff --git a/cmake/FindReadline.cmake b/cmake/FindReadline.cmake index de9ddc46d..f26911b26 100644 --- a/cmake/FindReadline.cmake +++ b/cmake/FindReadline.cmake @@ -66,7 +66,9 @@ check_function_exists(rl_copy_text HAVE_COPY_TEXT) check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION) if(NOT HAVE_COMPLETION_FUNCTION) - set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY}) + if (Readline_LIBRARY) + set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY}) + endif(Readline_LIBRARY) check_function_exists(rl_copy_text HAVE_COPY_TEXT_TC) check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION_TC) set(HAVE_COMPLETION_FUNCTION ${HAVE_COMPLETION_FUNCTION_TC}) diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 530f8e636..9100a8db3 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -32,7 +32,9 @@ #include "easylogging++.h" +#undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "default" + #define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes #define MAX_LOG_FILES 50 diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 5dbb7a478..9b03941ee 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -393,7 +393,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //ask it inside(!) critical region if we still able to go in event wait... size_t cnt = socket_.get_io_service().poll_one(); if(!cnt) - misc_utils::sleep_no_w(0); + misc_utils::sleep_no_w(1); } return true; @@ -889,7 +889,9 @@ POP_WARNINGS { try { - io_service_.run(); + size_t cnt = io_service_.run(); + if (cnt == 0) + misc_utils::sleep_no_w(1); } catch(const std::exception& ex) { diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index 8c6c1a64d..31495ae13 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -39,12 +39,14 @@ namespace misc_utils inline std::string transform_to_escape_sequence(const std::string& src) { static const char escaped[] = "\b\f\n\r\t\v\"\\/"; - if (std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)) == src.end()) + std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); + if (it == src.end()) return src; std::string res; res.reserve(2 * src.size()); - for(std::string::const_iterator it = src.begin(); it!=src.end(); ++it) + res.assign(src.begin(), it); + for(; it!=src.end(); ++it) { switch(*it) { diff --git a/contrib/epee/src/network_throttle-detail.cpp b/contrib/epee/src/network_throttle-detail.cpp index 28c85bb78..6f727a1cf 100644 --- a/contrib/epee/src/network_throttle-detail.cpp +++ b/contrib/epee/src/network_throttle-detail.cpp @@ -150,6 +150,7 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri m_any_packet_yet = false; m_slot_size = 1.0; // hard coded in few places m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle + m_last_sample_time = 0; } void network_throttle::set_name(const std::string &name) diff --git a/external/easylogging++/ea_config.h b/external/easylogging++/ea_config.h index 4c74925d3..4fb48ce3e 100644 --- a/external/easylogging++/ea_config.h +++ b/external/easylogging++/ea_config.h @@ -6,6 +6,7 @@ #define ELPP_NO_CHECK_MACROS #define ELPP_WINSOCK2 #define ELPP_NO_DEBUG_MACROS +#define ELPP_UTC_DATETIME #ifdef EASYLOGGING_CC #if !(!defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index a4bdad4cf..d57f3f3a0 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -1,13 +1,14 @@ // // Bismillah ar-Rahmaan ar-Raheem // -// Easylogging++ v9.94.1 +// Easylogging++ v9.96.5 // Cross-platform logging library for C++ applications // -// Copyright (c) 2017 muflihun.com +// Copyright (c) 2012-2018 Muflihun Labs +// Copyright (c) 2012-2018 @abumusamq // // This library is released under the MIT Licence. -// http://labs.muflihun.com/easyloggingpp/licence.php +// https://github.com/muflihun/easyloggingpp/blob/master/LICENSE // // https://github.com/muflihun/easyloggingpp // https://muflihun.github.io/easyloggingpp @@ -25,8 +26,95 @@ INITIALIZE_EASYLOGGINGPP namespace el { -// el::base::utils +// el::base namespace base { +// el::base::consts +namespace consts { + +// Level log values - These are values that are replaced in place of %level format specifier +// Extra spaces after format specifiers are only for readability purposes in log files +static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO"); +static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); +static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARNING"); +static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); +static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); +static const base::type::char_t* kVerboseLevelLogValue = + ELPP_LITERAL("VERBOSE"); // will become VERBOSE-x where x = verbose level +static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); +static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); +static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); +static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); +static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); +static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); +static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); +static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); +// Format specifiers - These are used to define log format +static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); +static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); +static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); +static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); +static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); +static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); +static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); +static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); +static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); +static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); +static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); +static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); +static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); +static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); +static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); +static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; +// Date/time +static const char* kDays[7] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; +static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; +static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", + "September", "October", "November", "December" + }; +static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; +static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; +static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; +static const int kYearBase = 1900; +static const char* kAm = "AM"; +static const char* kPm = "PM"; +// Miscellaneous constants + +static const char* kNullPointer = "nullptr"; +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +static const base::type::VerboseLevel kMaxVerboseLevel = 9; +static const char* kUnknownUser = "user"; +static const char* kUnknownHost = "unknown-host"; + + +//---------------- DEFAULT LOG FILE ----------------------- + +#if defined(ELPP_NO_DEFAULT_LOG_FILE) +# if ELPP_OS_UNIX +static const char* kDefaultLogFile = "/dev/null"; +# elif ELPP_OS_WINDOWS +static const char* kDefaultLogFile = "nul"; +# endif // ELPP_OS_UNIX +#elif defined(ELPP_DEFAULT_LOG_FILE) +static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; +#else +static const char* kDefaultLogFile = "myeasylog.log"; +#endif // defined(ELPP_NO_DEFAULT_LOG_FILE) + + +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +static const char* kDefaultLogFileParam = "--default-log-file"; +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kLoggingFlagsParam = "--logging-flags"; +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kValidLoggerIdSymbols = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; +static const char* kConfigurationComment = "##"; +static const char* kConfigurationLevel = "*"; +static const char* kConfigurationLoggerId = "--"; +} +// el::base::utils namespace utils { /// @brief Aborts application due with user-defined status @@ -303,11 +391,7 @@ void Configurations::set(Configuration* conf) { void Configurations::setToDefault(void) { setGlobally(ConfigurationType::Enabled, std::string("true"), true); -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) setGlobally(ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile), true); -#else - ELPP_UNUSED(base::consts::kDefaultLogFile); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) #if defined(ELPP_NO_LOG_TO_FILE) setGlobally(ConfigurationType::ToFile, std::string("false"), true); #else @@ -336,9 +420,7 @@ void Configurations::setRemainingToDefault(void) { #else unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("true")); #endif // defined(ELPP_NO_LOG_TO_FILE) -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) unsafeSetIfNotExist(Level::Global, ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile)); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) unsafeSetIfNotExist(Level::Global, ConfigurationType::ToStandardOutput, std::string("true")); unsafeSetIfNotExist(Level::Global, ConfigurationType::SubsecondPrecision, std::string("3")); unsafeSetIfNotExist(Level::Global, ConfigurationType::PerformanceTracking, std::string("true")); @@ -596,7 +678,6 @@ void Logger::configure(const Configurations& configurations) { if (m_typedConfigurations != nullptr) { Configurations* c = const_cast<Configurations*>(m_typedConfigurations->configurations()); if (c->hasConfiguration(Level::Global, ConfigurationType::Filename)) { - // This check is definitely needed for cases like ELPP_NO_DEFAULT_LOG_FILE flush(); } } @@ -640,7 +721,11 @@ void Logger::flush(Level level, base::type::fstream_t* fs) { } if (fs != nullptr) { fs->flush(); - m_unflushedCount.find(level)->second = 0; + std::unordered_map<Level, unsigned int>::iterator iter = m_unflushedCount.find(level); + if (iter != m_unflushedCount.end()) { + iter->second = 0; + } + Helpers::validateFileRolling(this, level); } } @@ -699,10 +784,9 @@ std::size_t File::getSizeOfFile(base::type::fstream_t* fs) { if (fs == nullptr) { return 0; } - std::streampos currPos = fs->tellg(); - fs->seekg(0, fs->end); + // Since the file stream is appended to or truncated, the current + // offset is the file size. std::size_t size = static_cast<std::size_t>(fs->tellg()); - fs->seekg(currPos); return size; } @@ -894,7 +978,10 @@ void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::st #endif // defined(ELPP_UNICODE) std::string& Str::toUpper(std::string& str) { - std::transform(str.begin(), str.end(), str.begin(), ::toupper); + std::transform(str.begin(), str.end(), str.begin(), + [](char c) { + return static_cast<char>(::toupper(c)); + }); return str; } @@ -1022,11 +1109,13 @@ const std::string OS::getBashOutput(const char* command) { char hBuff[4096]; if (fgets(hBuff, sizeof(hBuff), proc) != nullptr) { pclose(proc); - const size_t len = strlen(hBuff); - if (len > 0 && hBuff[len - 1] == '\n') { - hBuff[len- 1] = '\0'; + const std::size_t buffLen = strlen(hBuff); + if (buffLen > 0 && hBuff[buffLen - 1] == '\n') { + hBuff[buffLen- 1] = '\0'; } return std::string(hBuff); + } else { + pclose(proc); } return std::string(); #else @@ -1172,19 +1261,23 @@ unsigned long long DateTime::getTimeDifference(const struct timeval& endTime, co struct ::tm* DateTime::buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo) { #if ELPP_OS_UNIX time_t rawTime = currTime->tv_sec; - ::gmtime_r(&rawTime, timeInfo); + ::elpptime_r(&rawTime, timeInfo); return timeInfo; #else # if ELPP_COMPILER_MSVC ELPP_UNUSED(currTime); time_t t; +# if defined(_USE_32BIT_TIME_T) + _time32(&t); +# else _time64(&t); - gmtime_s(timeInfo, &t); +# endif + elpptime_s(timeInfo, &t); return timeInfo; # else // For any other compilers that don't have CRT warnings issue e.g, MinGW or TDM GCC- we use different method time_t rawTime = currTime->tv_sec; - struct tm* tmInf = gmtime(&rawTime); + struct tm* tmInf = elpptime(&rawTime); *timeInfo = *tmInf; return timeInfo; # endif // ELPP_COMPILER_MSVC @@ -1292,7 +1385,8 @@ bool CommandLineArgs::hasParamWithValue(const char* paramKey) const { } const char* CommandLineArgs::getParamValue(const char* paramKey) const { - return m_paramsWithValue.find(std::string(paramKey))->second.c_str(); + std::unordered_map<std::string, std::string>::const_iterator iter = m_paramsWithValue.find(std::string(paramKey)); + return iter != m_paramsWithValue.end() ? iter->second.c_str() : ""; } bool CommandLineArgs::hasParam(const char* paramKey) const { @@ -1641,10 +1735,11 @@ void TypedConfigurations::build(Configurations* configurations) { } else if (conf->configurationType() == ConfigurationType::PerformanceTracking) { setValue(Level::Global, getBool(conf->value()), &m_performanceTrackingMap); } else if (conf->configurationType() == ConfigurationType::MaxLogFileSize) { - setValue(conf->level(), static_cast<std::size_t>(getULong(conf->value())), &m_maxLogFileSizeMap); -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) - withFileSizeLimit.push_back(conf); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) + auto v = getULong(conf->value()); + setValue(conf->level(), static_cast<std::size_t>(v), &m_maxLogFileSizeMap); + if (v != 0) { + withFileSizeLimit.push_back(conf); + } } else if (conf->configurationType() == ConfigurationType::LogFlushThreshold) { setValue(conf->level(), static_cast<std::size_t>(getULong(conf->value())), &m_logFlushThresholdMap); } @@ -1718,12 +1813,6 @@ std::string TypedConfigurations::resolveFilename(const std::string& filename) { } void TypedConfigurations::insertFile(Level level, const std::string& fullFilename) { -#if defined(ELPP_NO_LOG_TO_FILE) - setValue(level, false, &m_toFileMap); - ELPP_UNUSED(fullFilename); - m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(nullptr))); - return; -#endif if (fullFilename.empty()) return; std::string resolvedFilename = resolveFilename(fullFilename); @@ -1861,8 +1950,10 @@ bool RegisteredLoggers::remove(const std::string& id) { if (id == base::consts::kDefaultLoggerId) { return false; } + // get has internal lock Logger* logger = base::utils::Registry<Logger, std::string>::get(id); if (logger != nullptr) { + // unregister has internal lock unregister(logger); } return true; @@ -1981,6 +2072,12 @@ void VRegistry::setCategories(const char* categories, bool clear) { m_categoriesString += ","; m_categoriesString += categories; + size_t n_fields = m_categories.size() + 1; + for (const char *ptr = categories; *ptr; ++ptr) + if (*ptr == ',') + ++n_fields; + m_categories.reserve(n_fields); + bool isCat = true; bool isLevel = false; std::stringstream ss; @@ -2042,7 +2139,7 @@ bool VRegistry::allowed(Level level, const char* category) { if (m_categories.empty() || category == nullptr) { return false; } else { - std::deque<std::pair<std::string, Level>>::const_reverse_iterator it = m_categories.rbegin(); + std::vector<std::pair<std::string, Level>>::const_reverse_iterator it = m_categories.rbegin(); for (; it != m_categories.rend(); ++it) { if (base::utils::Str::wildCardMatch(category, it->first.c_str())) { const int p = priority(it->second); @@ -2060,9 +2157,11 @@ bool VRegistry::allowed(base::type::VerboseLevel vlevel, const char* file) { if (m_modules.empty() || file == nullptr) { return vlevel <= m_level; } else { - std::map<std::string, base::type::VerboseLevel>::iterator it = m_modules.begin(); + char baseFilename[base::consts::kSourceFilenameMaxLength] = ""; + base::utils::File::buildBaseFilename(file, baseFilename); + std::unordered_map<std::string, base::type::VerboseLevel>::iterator it = m_modules.begin(); for (; it != m_modules.end(); ++it) { - if (base::utils::Str::wildCardMatch(file, it->first.c_str())) { + if (base::utils::Str::wildCardMatch(baseFilename, it->first.c_str())) { return vlevel <= it->second; } } @@ -2170,7 +2269,7 @@ Storage::~Storage(void) { } bool Storage::hasCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); return std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier) != m_customFormatSpecifiers.end(); } @@ -2179,12 +2278,12 @@ void Storage::installCustomFormatSpecifier(const CustomFormatSpecifier& customFo if (hasCustomFormatSpecifier(customFormatSpecifier.formatSpecifier())) { return; } - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); m_customFormatSpecifiers.push_back(customFormatSpecifier); } bool Storage::uninstallCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); std::vector<CustomFormatSpecifier>::iterator it = std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier); if (it != m_customFormatSpecifiers.end() && strcmp(formatSpecifier, it->formatSpecifier()) == 0) { @@ -2222,9 +2321,35 @@ void Storage::setApplicationArguments(int argc, char** argv) { #endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) } +} // namespace base + +// LogDispatchCallback +void LogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + base::threading::ScopedLock scopedLock(m_fileLocksMapLock); + std::string filename = data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level()); + auto lock = m_fileLocks.find(filename); + if (lock == m_fileLocks.end()) { + m_fileLocks.emplace(std::make_pair(filename, std::unique_ptr<base::threading::Mutex>(new base::threading::Mutex))); + } +#endif +} + +base::threading::Mutex& LogDispatchCallback::fileHandle(const LogDispatchData* data) { + auto it = m_fileLocks.find(data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level())); + return *(it->second.get()); +} + +namespace base { // DefaultLogDispatchCallback void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) +#if 0 + LogDispatchCallback::handle(data); + base::threading::ScopedLock scopedLock(fileHandle(data)); +#endif +#endif m_data = data; dispatch(m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog)); @@ -2475,6 +2600,8 @@ base::type::string_t DefaultLogBuilder::build(const LogMessage* logMessage, bool base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kMessageFormatSpecifier, logMessage->message()); } #if !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + el::base::threading::ScopedLock lock_(ELPP->customFormatSpecifiersLock()); + ELPP_UNUSED(lock_); for (std::vector<CustomFormatSpecifier>::const_iterator it = ELPP->customFormatSpecifiers()->begin(); it != ELPP->customFormatSpecifiers()->end(); ++it) { std::string fs(it->formatSpecifier()); @@ -2495,10 +2622,15 @@ void LogDispatcher::dispatch(void) { if (!m_proceed) { return; } +#ifndef ELPP_NO_GLOBAL_LOCK + // see https://github.com/muflihun/easyloggingpp/issues/580 + // global lock is turned off by default unless + // ELPP_NO_GLOBAL_LOCK is defined base::threading::ScopedLock scopedLock(ELPP->lock()); - base::TypedConfigurations* tc = m_logMessage.logger()->m_typedConfigurations; +#endif + base::TypedConfigurations* tc = m_logMessage->logger()->m_typedConfigurations; if (ELPP->hasFlag(LoggingFlag::StrictLogFileSizeCheck)) { - tc->validateFileRolling(m_logMessage.level(), ELPP->preRollOutCallback()); + tc->validateFileRolling(m_logMessage->level(), ELPP->preRollOutCallback()); } LogDispatchCallback* callback = nullptr; LogDispatchData data; @@ -2506,7 +2638,7 @@ void LogDispatcher::dispatch(void) { : ELPP->m_logDispatchCallbacks) { callback = h.second.get(); if (callback != nullptr && callback->enabled()) { - data.setLogMessage(&m_logMessage); + data.setLogMessage(m_logMessage); data.setDispatchAction(m_dispatchAction); callback->handle(&data); } @@ -2553,6 +2685,7 @@ Writer& Writer::construct(int count, const char* loggerIds, ...) { va_list loggersList; va_start(loggersList, loggerIds); const char* id = loggerIds; + m_loggerIds.reserve(count); for (int i = 0; i < count; ++i) { m_loggerIds.push_back(std::string(id)); id = va_arg(loggersList, const char*); @@ -2571,12 +2704,12 @@ void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool nee m_logger = ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically)); } if (m_logger == nullptr) { - ELPP->acquireLock(); - if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { - // Somehow default logger has been unregistered. Not good! Register again - ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + { + if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { + // Somehow default logger has been unregistered. Not good! Register again + ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + } } - ELPP->releaseLock(); // Need to unlock it for next writer Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) << "Logger [" << loggerId << "] is not registered yet!"; m_proceed = false; @@ -2637,8 +2770,13 @@ void Writer::processDispatch() { void Writer::triggerDispatch(void) { if (m_proceed) { - base::LogDispatcher(m_proceed, LogMessage(m_level, m_file, m_line, m_func, m_verboseLevel, - m_logger), m_dispatchAction).dispatch(); + if (m_msg == nullptr) { + LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, + m_logger); + base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); + } else { + base::LogDispatcher(m_proceed, m_msg, m_dispatchAction).dispatch(); + } } if (m_logger != nullptr) { m_logger->stream().str(ELPP_LITERAL("")); @@ -2651,7 +2789,7 @@ void Writer::triggerDispatch(void) { std::stringstream reasonStream; reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" << " If you wish to disable 'abort on fatal log' please use " - << "el::Helpers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; + << "el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; base::utils::abort(1, reasonStream.str()); } m_proceed = false; @@ -2767,18 +2905,19 @@ namespace debug { // StackTrace -StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const char* loc, const char* demang, const char* hex, - const char* addr) { - m_index = index; - m_location = std::string(loc); - m_demangled = std::string(demang); - m_hex = std::string(hex); - m_addr = std::string(addr); +StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, + const std::string& hex, + const std::string& addr) : + m_index(index), + m_location(loc), + m_demangled(demang), + m_hex(hex), + m_addr(addr) { } std::ostream& operator<<(std::ostream& ss, const StackTrace::StackTraceEntry& si) { - ss << "[" << si.m_index << "] " << si.m_location << (si.m_demangled.empty() ? "" : ":") << si.m_demangled - << (si.m_hex.empty() ? "" : "+") << si.m_hex << si.m_addr; + ss << "[" << si.m_index << "] " << si.m_location << (si.m_hex.empty() ? "" : "+") << si.m_hex << " " << si.m_addr << + (si.m_demangled.empty() ? "" : ":") << si.m_demangled; return ss; } @@ -2798,44 +2937,40 @@ void StackTrace::generateNew(void) { char** strings = backtrace_symbols(stack, size); if (size > kStackStart) { // Skip StackTrace c'tor and generateNew for (std::size_t i = kStackStart; i < size; ++i) { - char* mangName = nullptr; - char* hex = nullptr; - char* addr = nullptr; - for (char* c = strings[i]; *c; ++c) { - switch (*c) { - case '(': - mangName = c; - break; - case '+': - hex = c; - break; - case ')': - addr = c; - break; - default: - break; - } + std::string mangName; + std::string location; + std::string hex; + std::string addr; + + // entry: 2 crash.cpp.bin 0x0000000101552be5 _ZN2el4base5debug10StackTraceC1Ev + 21 + const std::string line(strings[i]); + auto p = line.find("_"); + if (p != std::string::npos) { + mangName = line.substr(p); + mangName = mangName.substr(0, mangName.find(" +")); + } + p = line.find("0x"); + if (p != std::string::npos) { + addr = line.substr(p); + addr = addr.substr(0, addr.find("_")); } // Perform demangling if parsed properly - if (mangName != nullptr && hex != nullptr && addr != nullptr && mangName < hex) { - *mangName++ = '\0'; - *hex++ = '\0'; - *addr++ = '\0'; + if (!mangName.empty()) { int status = 0; - char* demangName = abi::__cxa_demangle(mangName, 0, 0, &status); + char* demangName = abi::__cxa_demangle(mangName.data(), 0, 0, &status); // if demangling is successful, output the demangled function name if (status == 0) { // Success (see http://gcc.gnu.org/onlinedocs/libstdc++/libstdc++-html-USERS-4.3/a01696.html) - StackTraceEntry entry(i - 1, strings[i], demangName, hex, addr); + StackTraceEntry entry(i - 1, location, demangName, hex, addr); m_stack.push_back(entry); } else { // Not successful - we will use mangled name - StackTraceEntry entry(i - 1, strings[i], mangName, hex, addr); + StackTraceEntry entry(i - 1, location, mangName, hex, addr); m_stack.push_back(entry); } free(demangName); } else { - StackTraceEntry entry(i - 1, strings[i]); + StackTraceEntry entry(i - 1, line); m_stack.push_back(entry); } } @@ -2869,6 +3004,9 @@ static std::string crashReason(int sig) { } /// @brief Logs reason of crash from sig static void logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + if (sig == SIGINT && ELPP->hasFlag(el::LoggingFlag::IgnoreSigInt)) { + return; + } std::stringstream ss; ss << "CRASH HANDLED; "; ss << crashReason(sig); @@ -2947,7 +3085,6 @@ void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, c // Loggers Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); } @@ -2956,12 +3093,10 @@ void Loggers::setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr) { } bool Loggers::unregisterLogger(const std::string& identity) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->remove(identity); } bool Loggers::hasLogger(const std::string& identity) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->has(identity); } @@ -3131,11 +3266,11 @@ const std::string &Loggers::getFilenameCommonPrefix() { // VersionInfo const std::string VersionInfo::version(void) { - return std::string("9.94.1"); + return std::string("9.96.5"); } /// @brief Release date of current version const std::string VersionInfo::releaseDate(void) { - return std::string("25-02-2017 0813hrs"); + return std::string("07-09-2018 0950hrs"); } } // namespace el diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 8e8b7094b..046252a5b 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -1,18 +1,20 @@ // // Bismillah ar-Rahmaan ar-Raheem // -// Easylogging++ v9.94.1 +// Easylogging++ v9.96.5 // Single-header only, cross-platform logging library for C++ applications // -// Copyright (c) 2017 muflihun.com +// Copyright (c) 2012-2018 Muflihun Labs +// Copyright (c) 2012-2018 @abumusamq // // This library is released under the MIT Licence. -// http://labs.muflihun.com/easyloggingpp/licence.php +// https://github.com/muflihun/easyloggingpp/blob/master/LICENSE // // https://github.com/muflihun/easyloggingpp // https://muflihun.github.io/easyloggingpp // http://muflihun.com // + #ifndef EASYLOGGINGPP_H #define EASYLOGGINGPP_H #include "ea_config.h" @@ -94,7 +96,7 @@ #else # define ELPP_OS_MAC 0 #endif -#if (defined(__FreeBSD__)) +#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) # define ELPP_OS_FREEBSD 1 #else # define ELPP_OS_FREEBSD 0 @@ -114,13 +116,23 @@ #else # define ELPP_OS_SOLARIS 0 #endif +#if (defined(_AIX)) +# define ELPP_OS_AIX 1 +#else +# define ELPP_OS_AIX 0 +#endif +#if (defined(__NetBSD__)) +# define ELPP_OS_NETBSD 1 +#else +# define ELPP_OS_NETBSD 0 +#endif #if (defined(__DragonFly__)) # define ELPP_OS_DRAGONFLY 1 #else # define ELPP_OS_DRAGONFLY 0 #endif // Unix -#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_SOLARIS || ELPP_OS_DRAGONFLY || ELPP_OS_OPENBSD || ELPP_OS_NETBSD ) && (!ELPP_OS_WINDOWS)) +#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_NETBSD || ELPP_OS_SOLARIS || ELPP_OS_AIX || ELPP_OS_DRAGONFLY || ELPP_OS_OPENBSD) && (!ELPP_OS_WINDOWS)) # define ELPP_OS_UNIX 1 #else # define ELPP_OS_UNIX 0 @@ -205,7 +217,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre # define ELPP_INTERNAL_INFO(lvl, msg) #endif // (defined(ELPP_DEBUG_INFO)) #if (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) -# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_OPENBSD && !ELPP_OS_NETBSD) +# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_OPENBSD && !ELPP_OS_NETBSD && !ELPP_OS_ANDROID) # define ELPP_STACKTRACE 1 # else # define ELPP_STACKTRACE 0 @@ -386,7 +398,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #include <string> #include <vector> #include <map> -#include <deque> +#include <unordered_map> #include <utility> #include <functional> #include <algorithm> @@ -424,9 +436,6 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre # if defined(ELPP_LOG_STD_ARRAY) # include <array> # endif // defined(ELPP_LOG_STD_ARRAY) -# if defined(ELPP_LOG_UNORDERED_MAP) -# include <unordered_map> -# endif // defined(ELPP_LOG_UNORDERED_MAP) # if defined(ELPP_LOG_UNORDERED_SET) # include <unordered_set> # endif // defined(ELPP_UNORDERED_SET) @@ -461,6 +470,15 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre // For logging wxWidgets based classes & templates # include <wx/vector.h> #endif // defined(ELPP_WXWIDGETS_LOGGING) +#if defined(ELPP_UTC_DATETIME) +# define elpptime_r gmtime_r +# define elpptime_s gmtime_s +# define elpptime gmtime +#else +# define elpptime_r localtime_r +# define elpptime_s localtime_s +# define elpptime localtime +#endif // defined(ELPP_UTC_DATETIME) // Forward declarations namespace el { class Logger; @@ -585,6 +603,16 @@ enum class Level : base::type::EnumType { /// @brief Represents unknown level Unknown = 1010 }; +} // namespace el +namespace std { +template<> struct hash<el::Level> { + public: + std::size_t operator()(const el::Level& l) const { + return hash<el::base::type::EnumType> {}(static_cast<el::base::type::EnumType>(l)); + } +}; +} +namespace el { /// @brief Static class that contains helper functions for el::Level class LevelHelper : base::StaticClass { public: @@ -711,113 +739,41 @@ enum class LoggingFlag : base::type::EnumType { /// @brief Adds spaces b/w logs that separated by left-shift operator AutoSpacing = 8192, /// @brief Preserves time format and does not convert it to sec, hour etc (performance tracking only) - FixedTimeFormat = 16384 + FixedTimeFormat = 16384, + // @brief Ignore SIGINT or crash + IgnoreSigInt = 32768, }; namespace base { /// @brief Namespace containing constants used internally. namespace consts { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif -// Level log values - These are values that are replaced in place of %level format specifier -static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO "); -static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); -static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARN "); -static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); -static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); -static const base::type::char_t* kVerboseLevelLogValue = ELPP_LITERAL("VER"); -static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); -static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); -static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); -static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); -static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); -static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); -static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); -static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); -// Format specifiers - These are used to define log format -static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); -static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); -static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); -static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); -static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); -static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); -static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); -static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); -static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); -static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); -static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); -static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); -static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); -static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); -static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); -static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; -// Date/time -static const char* kDays[7] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; -static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; -static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", - "September", "October", "November", "December" - }; -static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; -static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; -static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; -static const int kYearBase = 1900; -static const char* kAm = "AM"; -static const char* kPm = "PM"; -// Miscellaneous constants +static const char kFormatSpecifierCharValue = 'v'; +static const char kFormatSpecifierChar = '%'; +static const unsigned int kMaxLogPerCounter = 100000; +static const unsigned int kMaxLogPerContainer = 100; +static const unsigned int kDefaultSubsecondPrecision = 3; + #ifdef ELPP_DEFAULT_LOGGER static const char* kDefaultLoggerId = ELPP_DEFAULT_LOGGER; #else static const char* kDefaultLoggerId = "default"; #endif + #ifdef ELPP_DEFAULT_PERFORMANCE_LOGGER static const char* kPerformanceLoggerId = ELPP_DEFAULT_PERFORMANCE_LOGGER; #else static const char* kPerformanceLoggerId = "performance"; #endif + #if defined(ELPP_SYSLOG) static const char* kSysLogLoggerId = "syslog"; #endif // defined(ELPP_SYSLOG) -static const char* kNullPointer = "nullptr"; -static const char kFormatSpecifierChar = '%'; -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const char kFormatSpecifierCharValue = 'v'; -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const unsigned int kMaxLogPerContainer = 100; -static const unsigned int kMaxLogPerCounter = 100000; -static const unsigned int kDefaultSubsecondPrecision = 3; -static const base::type::VerboseLevel kMaxVerboseLevel = 9; -static const char* kUnknownUser = "user"; -static const char* kUnknownHost = "unknown-host"; -#if defined(ELPP_DEFAULT_LOG_FILE) -static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; -#else -# if ELPP_OS_UNIX -# if ELPP_OS_ANDROID -static const char* kDefaultLogFile = "logs/myeasylog.log"; -# else -static const char* kDefaultLogFile = "logs/myeasylog.log"; -# endif // ELPP_OS_ANDROID -# elif ELPP_OS_WINDOWS -static const char* kDefaultLogFile = "logs\\myeasylog.log"; -# endif // ELPP_OS_UNIX -#endif // defined(ELPP_DEFAULT_LOG_FILE) -#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -static const char* kDefaultLogFileParam = "--default-log-file"; -#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) -static const char* kLoggingFlagsParam = "--logging-flags"; -#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) + #if ELPP_OS_WINDOWS static const char* kFilePathSeperator = "\\"; #else static const char* kFilePathSeperator = "/"; #endif // ELPP_OS_WINDOWS -static const char* kValidLoggerIdSymbols = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; -static const char* kConfigurationComment = "##"; -static const char* kConfigurationLevel = "*"; -static const char* kConfigurationLoggerId = "--"; + static const std::size_t kSourceFilenameMaxLength = 100; static const std::size_t kSourceLineMaxLength = 10; static const Level kPerformanceTrackerDefaultLevel = Level::Info; @@ -862,9 +818,6 @@ const struct { }, }; static const int kCrashSignalsCount = sizeof(kCrashSignals) / sizeof(kCrashSignals[0]); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif } // namespace consts } // namespace base typedef std::function<void(const char*, std::size_t)> PreRollOutCallback; @@ -1271,8 +1224,8 @@ class DateTime : base::StaticClass { base::TimestampUnit timestampUnit); - private: static struct ::tm* buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo); + private: static char* parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, std::size_t msec, const base::SubsecondPrecision* ssPrec); }; @@ -1311,7 +1264,7 @@ class CommandLineArgs { private: int m_argc; char** m_argv; - std::map<std::string, std::string> m_paramsWithValue; + std::unordered_map<std::string, std::string> m_paramsWithValue; std::vector<std::string> m_params; }; /// @brief Abstract registry (aka repository) that provides basic interface for pointer repository specified by T_Ptr type. @@ -1436,7 +1389,7 @@ class AbstractRegistry : public base::threading::ThreadSafe { /// of AbstractRegistry<T_Ptr, Container>. Any implementation of this class should be /// explicitly (by using lock functions) template <typename T_Ptr, typename T_Key = const char*> -class Registry : public AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>> { +class Registry : public AbstractRegistry<T_Ptr, std::unordered_map<T_Key, T_Ptr*>> { public: typedef typename Registry<T_Ptr, T_Key>::iterator iterator; typedef typename Registry<T_Ptr, T_Key>::const_iterator const_iterator; @@ -1486,8 +1439,8 @@ class Registry : public AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>> { void unregister(const T_Key& uniqKey) { T_Ptr* existing = get(uniqKey); if (existing != nullptr) { - base::utils::safeDelete(existing); this->list().erase(uniqKey); + base::utils::safeDelete(existing); } } @@ -1500,7 +1453,7 @@ class Registry : public AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>> { } private: - virtual void deepCopy(const AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>>& sr) ELPP_FINAL { + virtual void deepCopy(const AbstractRegistry<T_Ptr, std::unordered_map<T_Key, T_Ptr*>>& sr) ELPP_FINAL { for (const_iterator it = sr.cbegin(); it != sr.cend(); ++it) { registerNew(it->first, new T_Ptr(*it->second)); } @@ -1600,7 +1553,7 @@ class RegistryWithPred : public AbstractRegistry<T_Ptr, std::vector<T_Ptr*>> { class Utils { public: template <typename T, typename TPtr> - static bool installCallback(const std::string& id, std::map<std::string, TPtr>* mapT) { + static bool installCallback(const std::string& id, std::unordered_map<std::string, TPtr>* mapT) { if (mapT->find(id) == mapT->end()) { mapT->insert(std::make_pair(id, TPtr(new T()))); return true; @@ -1609,15 +1562,15 @@ class Utils { } template <typename T, typename TPtr> - static void uninstallCallback(const std::string& id, std::map<std::string, TPtr>* mapT) { + static void uninstallCallback(const std::string& id, std::unordered_map<std::string, TPtr>* mapT) { if (mapT->find(id) != mapT->end()) { mapT->erase(id); } } template <typename T, typename TPtr> - static T* callback(const std::string& id, std::map<std::string, TPtr>* mapT) { - typename std::map<std::string, TPtr>::iterator iter = mapT->find(id); + static T* callback(const std::string& id, std::unordered_map<std::string, TPtr>* mapT) { + typename std::unordered_map<std::string, TPtr>::iterator iter = mapT->find(id); if (iter != mapT->end()) { return static_cast<T*>(iter->second.get()); } @@ -1962,7 +1915,7 @@ class Configurations : public base::utils::RegistryWithPred<Configuration, Confi namespace base { typedef std::shared_ptr<base::type::fstream_t> FileStreamPtr; -typedef std::map<std::string, FileStreamPtr> LogStreamsReferenceMap; +typedef std::unordered_map<std::string, FileStreamPtr> LogStreamsReferenceMap; /// @brief Configurations with data types. /// /// @detail el::Configurations have string based values. This is whats used internally in order to read correct configurations. @@ -1999,16 +1952,16 @@ class TypedConfigurations : public base::threading::ThreadSafe { private: Configurations* m_configurations; - std::map<Level, bool> m_enabledMap; - std::map<Level, bool> m_toFileMap; - std::map<Level, std::string> m_filenameMap; - std::map<Level, bool> m_toStandardOutputMap; - std::map<Level, base::LogFormat> m_logFormatMap; - std::map<Level, base::SubsecondPrecision> m_subsecondPrecisionMap; - std::map<Level, bool> m_performanceTrackingMap; - std::map<Level, base::FileStreamPtr> m_fileStreamMap; - std::map<Level, std::size_t> m_maxLogFileSizeMap; - std::map<Level, std::size_t> m_logFlushThresholdMap; + std::unordered_map<Level, bool> m_enabledMap; + std::unordered_map<Level, bool> m_toFileMap; + std::unordered_map<Level, std::string> m_filenameMap; + std::unordered_map<Level, bool> m_toStandardOutputMap; + std::unordered_map<Level, base::LogFormat> m_logFormatMap; + std::unordered_map<Level, base::SubsecondPrecision> m_subsecondPrecisionMap; + std::unordered_map<Level, bool> m_performanceTrackingMap; + std::unordered_map<Level, base::FileStreamPtr> m_fileStreamMap; + std::unordered_map<Level, std::size_t> m_maxLogFileSizeMap; + std::unordered_map<Level, std::size_t> m_logFlushThresholdMap; base::LogStreamsReferenceMap* m_logStreamsReference; friend class el::Helpers; @@ -2018,21 +1971,21 @@ class TypedConfigurations : public base::threading::ThreadSafe { friend class el::base::LogDispatcher; template <typename Conf_T> - inline Conf_T getConfigByVal(Level level, const std::map<Level, Conf_T>* confMap, const char* confName) { + inline Conf_T getConfigByVal(Level level, const std::unordered_map<Level, Conf_T>* confMap, const char* confName) { base::threading::ScopedLock scopedLock(lock()); return unsafeGetConfigByVal(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope } template <typename Conf_T> - inline Conf_T& getConfigByRef(Level level, std::map<Level, Conf_T>* confMap, const char* confName) { + inline Conf_T& getConfigByRef(Level level, std::unordered_map<Level, Conf_T>* confMap, const char* confName) { base::threading::ScopedLock scopedLock(lock()); return unsafeGetConfigByRef(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope } template <typename Conf_T> - Conf_T unsafeGetConfigByVal(Level level, const std::map<Level, Conf_T>* confMap, const char* confName) { + Conf_T unsafeGetConfigByVal(Level level, const std::unordered_map<Level, Conf_T>* confMap, const char* confName) { ELPP_UNUSED(confName); - typename std::map<Level, Conf_T>::const_iterator it = confMap->find(level); + typename std::unordered_map<Level, Conf_T>::const_iterator it = confMap->find(level); if (it == confMap->end()) { try { return confMap->at(Level::Global); @@ -2047,9 +2000,9 @@ class TypedConfigurations : public base::threading::ThreadSafe { } template <typename Conf_T> - Conf_T& unsafeGetConfigByRef(Level level, std::map<Level, Conf_T>* confMap, const char* confName) { + Conf_T& unsafeGetConfigByRef(Level level, std::unordered_map<Level, Conf_T>* confMap, const char* confName) { ELPP_UNUSED(confName); - typename std::map<Level, Conf_T>::iterator it = confMap->find(level); + typename std::unordered_map<Level, Conf_T>::iterator it = confMap->find(level); if (it == confMap->end()) { try { return confMap->at(Level::Global); @@ -2063,14 +2016,15 @@ class TypedConfigurations : public base::threading::ThreadSafe { } template <typename Conf_T> - void setValue(Level level, const Conf_T& value, std::map<Level, Conf_T>* confMap, bool includeGlobalLevel = true) { + void setValue(Level level, const Conf_T& value, std::unordered_map<Level, Conf_T>* confMap, + bool includeGlobalLevel = true) { // If map is empty and we are allowed to add into generic level (Level::Global), do it! if (confMap->empty() && includeGlobalLevel) { confMap->insert(std::make_pair(Level::Global, value)); return; } // If same value exist in generic level already, dont add it to explicit level - typename std::map<Level, Conf_T>::iterator it = confMap->find(Level::Global); + typename std::unordered_map<Level, Conf_T>::iterator it = confMap->find(Level::Global); if (it != confMap->end() && it->second == value) { return; } @@ -2232,21 +2186,26 @@ class LogDispatchData { inline base::DispatchAction dispatchAction(void) const { return m_dispatchAction; } - private: - LogMessage* m_logMessage; - base::DispatchAction m_dispatchAction; - friend class base::LogDispatcher; - inline void setLogMessage(LogMessage* logMessage) { m_logMessage = logMessage; } inline void setDispatchAction(base::DispatchAction dispatchAction) { m_dispatchAction = dispatchAction; } + private: + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; + friend class base::LogDispatcher; + }; class LogDispatchCallback : public Callback<LogDispatchData> { + protected: + virtual void handle(const LogDispatchData* data); + base::threading::Mutex& fileHandle(const LogDispatchData* data); private: friend class base::LogDispatcher; + std::unordered_map<std::string, std::unique_ptr<base::threading::Mutex>> m_fileLocks; + base::threading::Mutex m_fileLocksMapLock; }; class PerformanceTrackingCallback : public Callback<PerformanceTrackingData> { private: @@ -2364,7 +2323,7 @@ inline void FUNCTION_NAME(const T&); std::string m_parentApplicationName; bool m_isConfigured; Configurations m_configurations; - std::map<Level, unsigned int> m_unflushedCount; + std::unordered_map<Level, unsigned int> m_unflushedCount; base::LogStreamsReferenceMap* m_logStreamsReference; LogBuilderPtr m_logBuilder; @@ -2470,7 +2429,7 @@ class RegisteredLoggers : public base::utils::Registry<Logger, std::string> { LogBuilderPtr m_defaultLogBuilder; Configurations m_defaultConfigurations; base::LogStreamsReferenceMap m_logStreamsReference; - std::map<std::string, base::type::LoggerRegistrationCallbackPtr> m_loggerRegistrationCallbacks; + std::unordered_map<std::string, base::type::LoggerRegistrationCallbackPtr> m_loggerRegistrationCallbacks; friend class el::base::Storage; void unsafeFlushAll(void); @@ -2508,7 +2467,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { bool allowed(base::type::VerboseLevel vlevel, const char* file); - inline const std::map<std::string, base::type::VerboseLevel>& modules(void) const { + inline const std::unordered_map<std::string, base::type::VerboseLevel>& modules(void) const { return m_modules; } @@ -2530,8 +2489,8 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { private: base::type::VerboseLevel m_level; base::type::EnumType* m_pFlags; - std::map<std::string, base::type::VerboseLevel> m_modules; - std::deque<std::pair<std::string, Level>> m_categories; + std::unordered_map<std::string, base::type::VerboseLevel> m_modules; + std::vector<std::pair<std::string, Level>> m_categories; std::map<std::string, int> m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; @@ -2718,6 +2677,10 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { return &m_customFormatSpecifiers; } + base::threading::Mutex& customFormatSpecifiersLock() { + return m_customFormatSpecifiersLock; + } + inline void setLoggingLevel(Level level) { m_loggingLevel = level; } @@ -2758,13 +2721,13 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { /// @brief Sets thread name for current thread. Requires std::thread inline void setThreadName(const std::string& name) { if (name.empty()) return; - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(m_threadNamesLock); m_threadNames[base::threading::getCurrentThreadId()] = name; } inline std::string getThreadName(const std::string& threadId) { - base::threading::ScopedLock scopedLock(lock()); - std::map<std::string, std::string>::const_iterator it = m_threadNames.find(threadId); + base::threading::ScopedLock scopedLock(m_threadNamesLock); + std::unordered_map<std::string, std::string>::const_iterator it = m_threadNames.find(threadId); if (it == m_threadNames.end()) { return threadId; } @@ -2784,10 +2747,12 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { #endif // ELPP_ASYNC_LOGGING base::utils::CommandLineArgs m_commandLineArgs; PreRollOutCallback m_preRollOutCallback; - std::map<std::string, base::type::LogDispatchCallbackPtr> m_logDispatchCallbacks; - std::map<std::string, base::type::PerformanceTrackingCallbackPtr> m_performanceTrackingCallbacks; - std::map<std::string, std::string> m_threadNames; + std::unordered_map<std::string, base::type::LogDispatchCallbackPtr> m_logDispatchCallbacks; + std::unordered_map<std::string, base::type::PerformanceTrackingCallbackPtr> m_performanceTrackingCallbacks; + std::unordered_map<std::string, std::string> m_threadNames; std::vector<CustomFormatSpecifier> m_customFormatSpecifiers; + base::threading::Mutex m_customFormatSpecifiersLock; + base::threading::Mutex m_threadNamesLock; Level m_loggingLevel; friend class el::Helpers; @@ -2830,7 +2795,7 @@ class AsyncDispatchWorker : public base::IWorker, public base::threading::Thread void run(void); void setContinueRunning(bool value) { - base::threading::ScopedLock scopedLock(m_continueRunningMutex); + base::threading::ScopedLock scopedLock(m_continueRunningLock); m_continueRunning = value; } @@ -2840,7 +2805,7 @@ class AsyncDispatchWorker : public base::IWorker, public base::threading::Thread private: std::condition_variable cv; bool m_continueRunning; - base::threading::Mutex m_continueRunningMutex; + base::threading::Mutex m_continueRunningLock; }; #endif // ELPP_ASYNC_LOGGING } // namespace base @@ -2852,9 +2817,9 @@ class DefaultLogBuilder : public LogBuilder { /// @brief Dispatches log messages class LogDispatcher : base::NoCopy { public: - LogDispatcher(bool proceed, LogMessage&& logMessage, base::DispatchAction dispatchAction) : + LogDispatcher(bool proceed, LogMessage* logMessage, base::DispatchAction dispatchAction) : m_proceed(proceed), - m_logMessage(std::move(logMessage)), + m_logMessage(logMessage), m_dispatchAction(std::move(dispatchAction)) { } @@ -2862,7 +2827,7 @@ class LogDispatcher : base::NoCopy { private: bool m_proceed; - LogMessage m_logMessage; + LogMessage* m_logMessage; base::DispatchAction m_dispatchAction; }; #if defined(ELPP_STL_LOGGING) @@ -3275,10 +3240,15 @@ class Writer : base::NoCopy { Writer(Level level, const char* file, base::type::LineNumber line, const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, base::type::VerboseLevel verboseLevel = 0) : - m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), + m_msg(nullptr), m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { } + Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) : + m_msg(msg), m_level(msg != nullptr ? msg->level() : Level::Unknown), + m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { + } + virtual ~Writer(void) { processDispatch(); } @@ -3321,6 +3291,7 @@ class Writer : base::NoCopy { Writer& construct(Logger* logger, bool needLock = true); Writer& construct(int count, const char* loggerIds, ...); protected: + LogMessage* m_msg; Level m_level; const char* m_file; const base::type::LineNumber m_line; @@ -3379,6 +3350,7 @@ void Logger::log_(Level level, int vlevel, const T& log) { base::DispatchAction::NormalLog, vlevel).construct(this, false) << log; } else { stream().str(ELPP_LITERAL("")); + releaseLock(); } } else { base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; @@ -3386,23 +3358,23 @@ void Logger::log_(Level level, int vlevel, const T& log) { } template <typename T, typename... Args> inline void Logger::log(Level level, const char* s, const T& value, const Args&... args) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(level, 0, s, value, args...); } template <typename T> inline void Logger::log(Level level, const T& log) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(level, 0, log); } # if ELPP_VERBOSE_LOG template <typename T, typename... Args> inline void Logger::verbose(int vlevel, const char* s, const T& value, const Args&... args) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(el::Level::Verbose, vlevel, s, value, args...); } template <typename T> inline void Logger::verbose(int vlevel, const T& log) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(el::Level::Verbose, vlevel, log); } # else @@ -3657,8 +3629,9 @@ class StackTrace : base::NoCopy { static const unsigned int kStackStart = 2; // We want to skip c'tor and StackTrace::generateNew() class StackTraceEntry { public: - StackTraceEntry(std::size_t index, const char* loc, const char* demang, const char* hex, const char* addr); - StackTraceEntry(std::size_t index, char* loc) : + StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, const std::string& hex, + const std::string& addr); + StackTraceEntry(std::size_t index, const std::string& loc) : m_index(index), m_location(loc) { } @@ -3843,6 +3816,11 @@ class Helpers : base::StaticClass { static inline const el::base::utils::CommandLineArgs* commandLineArgs(void) { return ELPP->commandLineArgs(); } + /// @brief Reserve space for custom format specifiers for performance + /// @see std::vector::reserve + static inline void reserveCustomFormatSpecifiers(std::size_t size) { + ELPP->m_customFormatSpecifiers.reserve(size); + } /// @brief Installs user defined format specifier and handler static inline void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { ELPP->installCustomFormatSpecifier(customFormatSpecifier); diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 84a083c26..1674c40dd 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2499,7 +2499,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 MDB_val_set(v, index); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); + throw1(OUTPUT_DNE(std::string("Attempting to get output pubkey by index, but key does not exist: amount " + + std::to_string(amount) + ", index " + std::to_string(index)).c_str())); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 7f92ecd87..eae078ea2 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -396,7 +396,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; std::cout << ENDL << ENDL; MINFO("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h); quit = 1; @@ -432,7 +432,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; } if (opt_verify) diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index beaad2abc..a8c46d661 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -304,7 +304,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem } if (m_cur_height % progress_interval == 0) { std::cout << refresh_string; - std::cout << "block " << m_cur_height << "/" << block_stop << std::flush; + std::cout << "block " << m_cur_height << "/" << block_stop << "\r" << std::flush; } } // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported. @@ -479,7 +479,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s bytes_read += count_bytes(import_file, progress_interval, blocks, quit); h += blocks; std::cout << "\r" << "block height: " << h-1 << - " " << + " \r" << std::flush; // std::cout << refresh_string; diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 75556cad9..b28a04f20 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -109,20 +109,8 @@ namespace tools assert(1 <= size && size <= sizeof(uint64_t)); uint64_t res = 0; - switch (9 - size) - { - case 1: res |= *data++; /* FALLTHRU */ - case 2: res <<= 8; res |= *data++; /* FALLTHRU */ - case 3: res <<= 8; res |= *data++; /* FALLTHRU */ - case 4: res <<= 8; res |= *data++; /* FALLTHRU */ - case 5: res <<= 8; res |= *data++; /* FALLTHRU */ - case 6: res <<= 8; res |= *data++; /* FALLTHRU */ - case 7: res <<= 8; res |= *data++; /* FALLTHRU */ - case 8: res <<= 8; res |= *data; break; - default: assert(false); - } - - return res; + memcpy(reinterpret_cast<uint8_t*>(&res) + sizeof(uint64_t) - size, data, size); + return SWAP64BE(res); } void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data) diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index 5d57b8af4..8a22a4b93 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -19,6 +19,7 @@ Issue Date: 20/12/2007 */ #include <stdint.h> +#include "common/int-util.h" #if defined(__cplusplus) extern "C" @@ -50,7 +51,7 @@ extern "C" #define state_out(y,x) so(y,x,0); so(y,x,1); so(y,x,2); so(y,x,3) #define round(rm,y,x,k) rm(y,x,k,0); rm(y,x,k,1); rm(y,x,k,2); rm(y,x,k,3) #define to_byte(x) ((x) & 0xff) -#define bval(x,n) to_byte((x) >> (8 * (n))) +#define bval(x,n) to_byte(SWAP32LE(x) >> (8 * (n))) #define fwd_var(x,r,c)\ ( r == 0 ? ( c == 0 ? s(x,0) : c == 1 ? s(x,1) : c == 2 ? s(x,2) : s(x,3))\ @@ -58,7 +59,7 @@ extern "C" : r == 2 ? ( c == 0 ? s(x,2) : c == 1 ? s(x,3) : c == 2 ? s(x,0) : s(x,1))\ : ( c == 0 ? s(x,3) : c == 1 ? s(x,0) : c == 2 ? s(x,1) : s(x,2))) -#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ four_tables(x,t_use(f,n),fwd_var,rf1,c)) +#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ SWAP32LE(four_tables(x,t_use(f,n),fwd_var,rf1,c))) #define sb_data(w) {\ w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), w(0xc5),\ diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 42f272e34..43ce32957 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -36,7 +36,14 @@ #include "keccak.h" void hash_permutation(union hash_state *state) { +#if BYTE_ORDER == LITTLE_ENDIAN keccakf((uint64_t*)state, 24); +#else + uint64_t le_state[25]; + memcpy_swap64le(le_state, state, 25); + keccakf(le_state, 24); + memcpy_swap64le(state, le_state, 25); +#endif } void hash_process(union hash_state *state, const uint8_t *buf, size_t count) { diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 40cfb0461..ed61e1017 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -47,8 +47,8 @@ #define INIT_SIZE_BLK 8 #define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_single_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ do if (variant == 1) \ @@ -1408,9 +1408,6 @@ static void (*const extra_hashes[4])(const void *, size_t, char *) = { hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein }; -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); - static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); } static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) { diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index b18ef1c5c..c4e10851e 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -328,7 +328,7 @@ bool parse_hash256(const std::string str_hash, crypto::hash& hash) bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); if (!res || buf.size() != sizeof(crypto::hash)) { - std::cout << "invalid hash format: <" << str_hash << '>' << std::endl; + MERROR("invalid hash format: " << str_hash); return false; } else diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index f05b25901..87a394918 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -56,12 +56,13 @@ static uint8_t get_block_version(const cryptonote::block &b) HardFork::HardFork(cryptonote::BlockchainDB &db, uint8_t original_version, uint64_t original_version_till_height, time_t forked_time, time_t update_time, uint64_t window_size, uint8_t default_threshold_percent): db(db), - original_version(original_version), - original_version_till_height(original_version_till_height), forked_time(forked_time), update_time(update_time), window_size(window_size), - default_threshold_percent(default_threshold_percent) + default_threshold_percent(default_threshold_percent), + original_version(original_version), + original_version_till_height(original_version_till_height), + current_fork_index(0) { if (window_size == 0) throw "window_size needs to be strictly positive"; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 77b6d0b69..e908c2012 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2530,7 +2530,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + if (((hf_version == HF_VERSION_MIN_MIXIN_10 || hf_version == HF_VERSION_MIN_MIXIN_10+1) && mixin != 10) || (hf_version >= HF_VERSION_MIN_MIXIN_10+2 && mixin > 10)) { MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); tvc.m_low_mixin = true; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 4b806c282..c405c996a 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1716,7 +1716,8 @@ namespace cryptonote for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) { unsigned int b = 0; - for (time_t ts: timestamps) b += ts >= now - static_cast<time_t>(seconds[n]); + const time_t time_boundary = now - static_cast<time_t>(seconds[n]); + for (time_t ts: timestamps) b += ts >= time_boundary; const double p = probability(b, seconds[n] / DIFFICULTY_TARGET_V2); MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")"); if (p < threshold) diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 07c03fc66..f55cbb15d 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -212,9 +212,10 @@ namespace trezor { tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) { + CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); size_t num_tx = unsigned_tx.txes.size(); signed_tx.key_images.clear(); - signed_tx.key_images.resize(unsigned_tx.transfers.size()); + signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { std::shared_ptr<protocol::tx::Signer> signer; @@ -360,4 +361,4 @@ namespace trezor { } #endif //WITH_DEVICE_TREZOR -}}
\ No newline at end of file +}} diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 496e26858..94071c23c 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -335,6 +335,7 @@ namespace crypto return false; } + w[0] = SWAP32LE(w[0]); dst.append((const char*)&w[0], 4); // copy 4 bytes to position memwipe(w, sizeof(w)); } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 8b4c27e3e..8fc42b7e3 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -27,11 +27,11 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set(rpc_base_sources - rpc_args.cpp - rpc_handler.cpp) + rpc_args.cpp) set(rpc_sources core_rpc_server.cpp + rpc_handler.cpp instanciations) set(daemon_messages_sources @@ -44,10 +44,10 @@ set(daemon_rpc_server_sources set(rpc_base_headers - rpc_args.h - rpc_handler.h) + rpc_args.h) -set(rpc_headers) +set(rpc_headers + rpc_handler.cpp) set(daemon_rpc_server_headers) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 574f6c126..c2e71bef8 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -701,7 +701,7 @@ namespace cryptonote res.status = "Failed"; res.reason = ""; if ((res.low_mixin = tvc.m_low_mixin)) - add_reason(res.reason, "ring size too small"); + add_reason(res.reason, "bad ring size"); if ((res.double_spend = tvc.m_double_spend)) add_reason(res.reason, "double spend"); if ((res.invalid_input = tvc.m_invalid_input)) @@ -802,7 +802,14 @@ namespace cryptonote bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) { PERF_TIMER(on_stop_mining); - if(!m_core.get_miner().stop()) + cryptonote::miner &miner= m_core.get_miner(); + if(!miner.is_mining()) + { + res.status = "Mining never started"; + LOG_PRINT_L0(res.status); + return true; + } + if(!miner.stop()) { res.status = "Failed, mining not stopped"; LOG_PRINT_L0(res.status); @@ -1011,7 +1018,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= h) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(h) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(h) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); } res = string_tools::pod_to_hex(m_core.get_block_id_by_height(h)); return true; @@ -1458,7 +1465,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } crypto::hash block_hash = m_core.get_block_id_by_height(req.height); @@ -1503,7 +1510,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } block_hash = m_core.get_block_id_by_height(req.height); diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 7980e8953..8b1af9c12 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -1192,7 +1192,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, incoming_connections_count, info.incoming_connections_count); INSERT_INTO_JSON_OBJECT(val, doc, white_peerlist_size, info.white_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, grey_peerlist_size, info.grey_peerlist_size); + INSERT_INTO_JSON_OBJECT(val, doc, mainnet, info.mainnet); INSERT_INTO_JSON_OBJECT(val, doc, testnet, info.testnet); + INSERT_INTO_JSON_OBJECT(val, doc, stagenet, info.stagenet); INSERT_INTO_JSON_OBJECT(val, doc, nettype, info.nettype); INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); @@ -1221,7 +1223,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.incoming_connections_count, incoming_connections_count); GET_FROM_JSON_OBJECT(val, info.white_peerlist_size, white_peerlist_size); GET_FROM_JSON_OBJECT(val, info.grey_peerlist_size, grey_peerlist_size); + GET_FROM_JSON_OBJECT(val, info.mainnet, mainnet); GET_FROM_JSON_OBJECT(val, info.testnet, testnet); + GET_FROM_JSON_OBJECT(val, info.stagenet, stagenet); GET_FROM_JSON_OBJECT(val, info.nettype, nettype); GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index c31cdebde..e292f85dd 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -53,6 +53,7 @@ target_link_libraries(simplewallet ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} + ${Boost_LOCALE_LIBRARY} ${ICU_LIBRARIES} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 58d4cdced..702ff22cb 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -2547,6 +2547,7 @@ simple_wallet::simple_wallet() tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), + tr("rescan_bc [hard]"), tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), @@ -4298,15 +4299,15 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init) +bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bool is_init) { if (!try_connect_to_daemon(is_init)) return true; LOCK_IDLE_SCOPE(); - if (reset) - m_wallet->rescan_blockchain(false); + if (reset != ResetNone) + m_wallet->rescan_blockchain(reset == ResetHard, false); #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4385,7 +4386,7 @@ bool simple_wallet::refresh(const std::vector<std::string>& args) start_height = 0; } } - return refresh_main(start_height, false); + return refresh_main(start_height, ResetNone); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::show_balance_unlocked(bool detailed) @@ -4483,7 +4484,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args tools::wallet2::transfer_container transfers; m_wallet->get_transfers(transfers); - bool transfers_found = false; + size_t transfers_found = 0; for (const auto& td : transfers) { if (!filter || available != td.m_spent) @@ -4496,7 +4497,6 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args if (verbose) verbose_string = (boost::format("%68s%68s") % tr("pubkey") % tr("key image")).str(); message_writer() << boost::format("%21s%8s%12s%8s%16s%68s%16s%s") % tr("amount") % tr("spent") % tr("unlocked") % tr("ringct") % tr("global index") % tr("tx id") % tr("addr index") % verbose_string; - transfers_found = true; } std::string verbose_string; if (verbose) @@ -4511,6 +4511,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args td.m_txid % td.m_subaddr_index.minor % verbose_string; + ++transfers_found; } } @@ -4529,6 +4530,10 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args success_msg_writer() << tr("No incoming unavailable transfers"); } } + else + { + success_msg_writer() << boost::format("Found %u/%u transfers") % transfers_found % transfers.size(); + } return true; } @@ -6105,8 +6110,8 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) { std::string extra_message; - if (!txs.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.size()).str(); + if (!txs.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.second.size()).str(); return accept_loaded_tx([&txs](){return txs.txes.size();}, [&txs](size_t n)->const tools::wallet2::tx_construction_data&{return txs.txes[n];}, extra_message); } //---------------------------------------------------------------------------------------------------- @@ -6859,7 +6864,8 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); const std::string type = pd.m_coinbase ? tr("block") : tr("in"); - output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_green, type, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); + const bool unlocked = m_wallet->is_tx_spendtime_unlocked(pd.m_unlock_time, pd.m_block_height); + output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_green, type, (boost::format("%8.8s %25.25s %20.20s %s %s %d %s %s") % (unlocked ? "unlocked" : "locked") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); } } @@ -6892,7 +6898,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); - output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_magenta, tr("out"), (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); + output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_magenta, tr("out"), (boost::format("%8.8s %25.25s %20.20s %s %s %14.14s %s %s - %s") % "-" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); } } @@ -6921,7 +6927,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string double_spend_note; if (i->second.m_double_spend_seen) double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); - message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); + message_writer() << (boost::format("%8.8s %6.6s %8.8s %25.25s %20.20s %s %s %d %s %s%s") % "pool" % "in" % "locked" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); } } catch (const std::exception& e) @@ -6944,7 +6950,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string note = m_wallet->get_tx_note(i->first); bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; if ((failed && is_failed) || (!is_failed && pending)) { - message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); + message_writer() << (boost::format("%8.8s %6.6s %8.8s %25.25s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % "-" % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); } } } @@ -7092,15 +7098,29 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { - message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); - message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); - std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); - if(!std::cin.eof()) + bool hard = false; + if (!args_.empty()) { - if (!command_line::is_yes(confirm)) + if (args_[0] != "hard") + { + fail_msg_writer() << tr("usage: rescan_bc [hard]"); return true; + } + hard = true; + } + + if (hard) + { + message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); + message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } } - return refresh_main(0, true); + return refresh_main(0, hard ? ResetHard : ResetSoft, true); } //---------------------------------------------------------------------------------------------------- void simple_wallet::wallet_idle_thread() @@ -7148,7 +7168,7 @@ bool simple_wallet::run() // check and display warning, but go on anyway try_connect_to_daemon(); - refresh_main(0, false, true); + refresh_main(0, ResetNone, true); m_auto_refresh_enabled = m_wallet->auto_refresh(); m_idle_thread = boost::thread([&]{wallet_idle_thread();}); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 7d813ceb0..26d51a431 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -83,6 +83,9 @@ namespace cryptonote std::string get_commands_str(); std::string get_command_usage(const std::vector<std::string> &args); private: + + enum ResetType { ResetNone, ResetSoft, ResetHard }; + bool handle_command_line(const boost::program_options::variables_map& vm); bool run_console_handler(); @@ -189,7 +192,7 @@ namespace cryptonote bool show_transfers(const std::vector<std::string> &args); bool unspent_outputs(const std::vector<std::string> &args); bool rescan_blockchain(const std::vector<std::string> &args); - bool refresh_main(uint64_t start_height, bool reset = false, bool is_init = false); + bool refresh_main(uint64_t start_height, ResetType reset, bool is_init = false); bool set_tx_note(const std::vector<std::string> &args); bool get_tx_note(const std::vector<std::string> &args); bool set_description(const std::vector<std::string> &args); diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index 4932dd4b6..4e3fb1ae5 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -120,7 +120,8 @@ if (BUILD_GUI_DEPS) ringct ringct_basic checkpoints - version) + version + device_trezor) foreach(lib ${libs_to_merge}) list(APPEND objlibs $<TARGET_OBJECTS:obj_${lib}>) # matches naming convention in src/CMakeLists.txt diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index ddf2d74ff..7cd3b65bb 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -507,7 +507,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas auto key_images = m_wallet->export_key_images(); uint64_t spent = 0; uint64_t unspent = 0; - view_wallet->import_key_images(key_images,spent,unspent,false); + view_wallet->import_key_images(key_images.second, key_images.first, spent, unspent, false); clearStatus(); } catch (const std::exception &e) { LOG_ERROR("Error creating view only wallet: " << e.what()); @@ -1051,8 +1051,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file // Check tx data and construct confirmation message std::string extra_message; - if (!transaction->m_unsigned_tx_set.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.size()).str(); + if (!transaction->m_unsigned_tx_set.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.second.size()).str(); transaction->checkLoadedTx([&transaction](){return transaction->m_unsigned_tx_set.txes.size();}, [&transaction](size_t n)->const tools::wallet2::tx_construction_data&{return transaction->m_unsigned_tx_set.txes[n];}, extra_message); setStatus(transaction->status(), transaction->errorString()); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 439840738..6f919d12c 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -69,6 +69,7 @@ using namespace epee; #include "common/base58.h" #include "common/dns_utils.h" #include "common/notify.h" +#include "common/perf_timer.h" #include "ringct/rctSigs.h" #include "ringdb.h" #include "device/device_cold.hpp" @@ -112,11 +113,11 @@ using namespace cryptonote; #define SUBADDRESS_LOOKAHEAD_MAJOR 50 #define SUBADDRESS_LOOKAHEAD_MINOR 200 -#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\002" +#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\003" #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\004" #define SEGREGATION_FORK_HEIGHT 99999999 #define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 @@ -1605,6 +1606,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -3149,13 +3151,22 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); keys_file_data.account_data = cipher; - unlock_keys_file(); + std::string tmp_file_name = keys_file_name + ".new"; std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); - r = r && epee::file_io_utils::save_string_to_file(keys_file_name, buf); //and never touch wallet_keys_file again, only read - CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << keys_file_name); + r = r && epee::file_io_utils::save_string_to_file(tmp_file_name, buf); + CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); + + unlock_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); lock_keys_file(); + if (e) { + boost::filesystem::remove(tmp_file_name); + LOG_ERROR("failed to update wallet keys file " << keys_file_name); + return false; + } + return true; } //---------------------------------------------------------------------------------------------------- @@ -5094,11 +5105,27 @@ void wallet2::rescan_spent() } } //---------------------------------------------------------------------------------------------------- -void wallet2::rescan_blockchain(bool refresh) +void wallet2::rescan_blockchain(bool hard, bool refresh) { - clear(); + if(hard) + { + clear(); + setup_new_blockchain(); + } + else + { + m_blockchain.clear(); + m_transfers.clear(); + m_key_images.clear(); + m_pub_keys.clear(); + m_scanned_pool_txs[0].clear(); + m_scanned_pool_txs[1].clear(); - setup_new_blockchain(); + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + } if (refresh) this->refresh(false); @@ -5465,7 +5492,7 @@ std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) c txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = export_outputs(); // save as binary std::ostringstream oss; boost::archive::portable_binary_oarchive ar(oss); @@ -7195,6 +7222,9 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> break; } } + bool use_histogram = amount != 0 || !has_rct_distribution; + if (!use_histogram) + num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; // make sure the real outputs we asked for are really included, along // with the correct key and mask: this guards against an active attack @@ -7287,6 +7317,20 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> outs.push_back(v); } } + + // save those outs in the ringdb for reuse + for (size_t i = 0; i < selected_transfers.size(); ++i) + { + const size_t idx = selected_transfers[i]; + THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "selected_transfers entry out of range"); + const transfer_details &td = m_transfers[idx]; + std::vector<uint64_t> ring; + ring.reserve(outs[i].size()); + for (const auto &e: outs[i]) + ring.push_back(std::get<0>(e)); + if (!set_ring(td.m_key_image, ring, false)) + MERROR("Failed to set ring for " << td.m_key_image); + } } template<typename T> @@ -7958,6 +8002,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -9131,7 +9176,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_ { txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = std::make_pair(0, m_transfers); auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); @@ -9162,7 +9207,7 @@ uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { dev_cold->ki_sync(&wallet_shim, m_transfers, ski); - return import_key_images(ski, spent, unspent); + return import_key_images(ski, 0, spent, unspent); } //---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const @@ -10524,31 +10569,45 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { - std::vector<std::pair<crypto::key_image, crypto::signature>> ski = export_key_images(); + PERF_TIMER(export_key_images); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + const uint32_t offset = ski.first; std::string data; + data.reserve(4 + ski.second.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.resize(4); + data[0] = offset & 0xff; + data[1] = (offset >> 8) & 0xff; + data[2] = (offset >> 16) & 0xff; + data[3] = (offset >> 24) & 0xff; data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - for (const auto &i: ski) + for (const auto &i: ski.second) { data += std::string((const char *)&i.first, sizeof(crypto::key_image)); data += std::string((const char *)&i.second, sizeof(crypto::signature)); } // encrypt data, keep magic plaintext + PERF_TIMER(export_key_images_encrypt); std::string ciphertext = encrypt_with_view_secret_key(data); return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key_images() const +std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images() const { + PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; - ski.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) + ++offset; + + ski.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; @@ -10592,11 +10651,12 @@ std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key ski.push_back(std::make_pair(td.m_key_image, signature)); } - return ski; + return std::make_pair(offset, ski); } uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) { + PERF_TIMER(import_key_images_fsu); std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -10610,6 +10670,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent try { + PERF_TIMER(import_key_images_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10617,15 +10678,17 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt ") + filename + ": " + e.what()); } - const size_t headerlen = 2 * sizeof(crypto::public_key); + const size_t headerlen = 4 + 2 * sizeof(crypto::public_key); THROW_WALLET_EXCEPTION_IF(data.size() < headerlen, error::wallet_internal_error, std::string("Bad data size from file ") + filename); - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const uint32_t offset = (uint8_t)data[0] | (((uint8_t)data[1]) << 8) | (((uint8_t)data[2]) << 16) | (((uint8_t)data[3]) << 24); + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[4]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[4 + sizeof(crypto::public_key)]; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string( "Key images from ") + filename + " are for a different account"); } + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); const size_t record_size = sizeof(crypto::key_image) + sizeof(crypto::signature); THROW_WALLET_EXCEPTION_IF((data.size() - headerlen) % record_size, @@ -10642,28 +10705,33 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent ski.push_back(std::make_pair(key_image, signature)); } - return import_key_images(ski, spent, unspent); + return import_key_images(ski, offset, spent, unspent); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) +uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent) { + PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); + THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size() - offset, error::wallet_internal_error, "The blockchain is out of date compared to the signed key images"); - if (signed_key_images.empty()) + if (signed_key_images.empty() && offset == 0) { spent = 0; unspent = 0; return 0; } + req.key_images.reserve(signed_key_images.size()); + + PERF_TIMER_START(import_key_images_A); for (size_t n = 0; n < signed_key_images.size(); ++n) { - const transfer_details &td = m_transfers[n]; + const transfer_details &td = m_transfers[n + offset]; const crypto::key_image &key_image = signed_key_images[n].first; const crypto::signature &signature = signed_key_images[n].second; @@ -10674,30 +10742,37 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const cryptonote::txout_to_key &o = boost::get<cryptonote::txout_to_key>(out.target); const crypto::public_key pkey = o.key; - std::vector<const crypto::public_key*> pkeys; - pkeys.push_back(&pkey); - THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n) + "/" - + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); - - THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast<std::string>(n) + "/" - + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) - + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + if (!td.m_key_image_known || !(key_image == td.m_key_image)) + { + std::vector<const crypto::public_key*> pkeys; + pkeys.push_back(&pkey); + THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n + offset) + "/" + + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); + THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), + error::signature_check_failed, boost::lexical_cast<std::string>(n + offset) + "/" + + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } req.key_images.push_back(epee::string_tools::pod_to_hex(key_image)); } + PERF_TIMER_STOP(import_key_images_A); + PERF_TIMER_START(import_key_images_B); for (size_t n = 0; n < signed_key_images.size(); ++n) { - m_transfers[n].m_key_image = signed_key_images[n].first; - m_key_images[m_transfers[n].m_key_image] = n; - m_transfers[n].m_key_image_known = true; - m_transfers[n].m_key_image_partial = false; + m_transfers[n + offset].m_key_image = signed_key_images[n].first; + m_key_images[m_transfers[n + offset].m_key_image] = n + offset; + m_transfers[n + offset].m_key_image_known = true; + m_transfers[n + offset].m_key_image_requested = false; + m_transfers[n + offset].m_key_image_partial = false; } + PERF_TIMER_STOP(import_key_images_B); if(check_spent) { + PERF_TIMER(import_key_images_RPC); m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -10709,7 +10784,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(signed_key_images.size())); for (size_t n = 0; n < daemon_resp.spent_status.size(); ++n) { - transfer_details &td = m_transfers[n]; + transfer_details &td = m_transfers[n + offset]; td.m_spent = daemon_resp.spent_status[n] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT; } } @@ -10720,6 +10795,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag // was created by sweep_all, so we can't know the spent height and other detailed info. std::unordered_map<crypto::key_image, crypto::hash> spent_key_images; + PERF_TIMER_START(import_key_images_C); for (const transfer_details &td: m_transfers) { for (const cryptonote::txin_v& in : td.m_tx.vin) @@ -10728,10 +10804,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid)); } } + PERF_TIMER_STOP(import_key_images_C); + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { - transfer_details &td = m_transfers[i]; + const transfer_details &td = m_transfers[i + offset]; uint64_t amount = td.amount(); if (td.m_spent) spent += amount; @@ -10749,6 +10827,8 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag spent_txids.insert(skii->second); } } + PERF_TIMER_STOP(import_key_images_D); + MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); if (check_spent) @@ -10758,8 +10838,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res; gettxs_req.decode_as_json = false; gettxs_req.prune = false; + gettxs_req.txs_hashes.reserve(spent_txids.size()); for (const crypto::hash& spent_txid : spent_txids) gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid)); + + + PERF_TIMER_START(import_key_images_E); m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -10767,8 +10851,10 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag THROW_WALLET_EXCEPTION_IF(gettxs_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size())); + PERF_TIMER_STOP(import_key_images_E); // process each outgoing tx + PERF_TIMER_START(import_key_images_F); auto spent_txid = spent_txids.begin(); hw::device &hwdev = m_account.get_device(); for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs) @@ -10864,7 +10950,9 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag ++spent_txid; } + PERF_TIMER_STOP(import_key_images_F); + PERF_TIMER_START(import_key_images_G); for (size_t n : swept_transfers) { const transfer_details& td = m_transfers[n]; @@ -10875,6 +10963,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown m_confirmed_txs.insert(std::make_pair(spent_txid, pd)); } + PERF_TIMER_STOP(import_key_images_G); } return m_transfers[signed_key_images.size() - 1].m_block_height; @@ -10895,6 +10984,7 @@ bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) td.m_key_image = key_images[i]; m_key_images[m_transfers[i].m_key_image] = i; td.m_key_image_known = true; + td.m_key_image_requested = false; td.m_key_image_partial = false; m_pub_keys[m_transfers[i].get_public_key()] = i; } @@ -10960,50 +11050,87 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- -std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const +std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs() const { + PERF_TIMER(export_outputs); std::vector<tools::wallet2::transfer_details> outs; - outs.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + ++offset; + + outs.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; outs.push_back(td); } - return outs; + return std::make_pair(offset, outs); } //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const { - std::vector<tools::wallet2::transfer_details> outs = export_outputs(); + PERF_TIMER(export_outputs_to_str); std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << outs; + const auto& outputs = export_outputs(); + ar << outputs; std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + PERF_TIMER(export_outputs_encryption); std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs) +size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) { - m_transfers.clear(); - m_transfers.reserve(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) + PERF_TIMER(import_outputs); + + THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error, + "Imported outputs omit more outputs that we know of"); + + const size_t offset = outputs.first; + const size_t original_size = m_transfers.size(); + m_transfers.resize(offset + outputs.second.size()); + for (size_t i = 0; i < offset; ++i) + m_transfers[i].m_key_image_requested = false; + for (size_t i = 0; i < outputs.second.size(); ++i) { - transfer_details td = outputs[i]; + transfer_details td = outputs.second[i]; + + // skip those we've already imported, or which have different data + if (i + offset < original_size) + { + // compare the data used to create the key image below + const transfer_details &org_td = m_transfers[i + offset]; + if (!org_td.m_key_image_known) + goto process; +#define CMPF(f) if (!(td.f == org_td.f)) goto process + CMPF(m_txid); + CMPF(m_key_image); + CMPF(m_internal_output_index); +#undef CMPF + if (!(get_transaction_prefix_hash(td.m_tx) == get_transaction_prefix_hash(org_td.m_tx))) + goto process; + + // copy anyway, since the comparison does not include ancillary fields which may have changed + m_transfers[i + offset] = std::move(td); + continue; + } + +process: // the hot wallet wouldn't have known about key images (except if we already exported them) cryptonote::keypair in_ephemeral; - THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast<std::string>(i)); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast<std::string>(i + offset)); crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); @@ -11014,13 +11141,14 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; + td.m_key_image_requested = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, - error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i)); + error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i + offset)); - m_key_images[td.m_key_image] = m_transfers.size(); - m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(std::move(td)); + m_key_images[td.m_key_image] = i + offset; + m_pub_keys[td.get_public_key()] = i + offset; + m_transfers[i + offset] = std::move(td); } return m_transfers.size(); @@ -11028,6 +11156,7 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail //---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs_from_str(const std::string &outputs_st) { + PERF_TIMER(import_outputs_from_str); std::string data = outputs_st; const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) @@ -11037,6 +11166,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) try { + PERF_TIMER(import_outputs_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -11063,7 +11193,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) std::string body(data, headerlen); std::stringstream iss; iss << body; - std::vector<tools::wallet2::transfer_details> outputs; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; try { boost::archive::portable_binary_iarchive ar(iss); @@ -11255,6 +11385,7 @@ void wallet2::update_multisig_rescan_info(const std::vector<std::vector<rct::key m_key_images.erase(td.m_key_image); td.m_key_image = get_multisig_composite_key_image(n); td.m_key_image_known = true; + td.m_key_image_requested = false; td.m_key_image_partial = false; td.m_multisig_k = multisig_k[n]; m_key_images[td.m_key_image] = n; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 7d78a3fee..eb0763131 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -230,7 +230,7 @@ namespace tools bool error; boost::optional<cryptonote::subaddress_receive_info> received; - tx_scan_info_t(): money_transfered(0), error(true) {} + tx_scan_info_t(): amount(0), money_transfered(0), error(true) {} }; struct transfer_details @@ -247,6 +247,7 @@ namespace tools uint64_t m_amount; bool m_rct; bool m_key_image_known; + bool m_key_image_requested; size_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; @@ -270,6 +271,7 @@ namespace tools FIELD(m_amount) FIELD(m_rct) FIELD(m_key_image_known) + FIELD(m_key_image_requested) FIELD(m_pk_index) FIELD(m_subaddr_index) FIELD(m_key_image_partial) @@ -417,7 +419,7 @@ namespace tools struct unsigned_tx_set { std::vector<tx_construction_data> txes; - wallet2::transfer_container transfers; + std::pair<size_t, wallet2::transfer_container> transfers; }; struct signed_tx_set @@ -786,7 +788,7 @@ namespace tools uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); - void rescan_blockchain(bool refresh = true); + void rescan_blockchain(bool hard, bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; @@ -1071,9 +1073,9 @@ namespace tools bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::vector<tools::wallet2::transfer_details> export_outputs() const; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs() const; std::string export_outputs_to_str() const; - size_t import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs); + size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1081,8 +1083,8 @@ namespace tools std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename) const; - std::vector<std::pair<crypto::key_image, crypto::signature>> export_key_images() const; - uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images() const; + uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector<crypto::key_image> key_images); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; @@ -1199,6 +1201,8 @@ namespace tools void set_tx_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_tx_notify = notify; } + bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; + private: /*! * \brief Stores wallet information to wallet file. @@ -1218,7 +1222,6 @@ namespace tools void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; - bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); @@ -1396,7 +1399,7 @@ namespace tools }; } BOOST_CLASS_VERSION(tools::wallet2, 26) -BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) +BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 10) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) @@ -1454,6 +1457,10 @@ namespace boost x.m_multisig_k.clear(); x.m_multisig_info.clear(); } + if (ver < 10) + { + x.m_key_image_requested = false; + } } template <class Archive> @@ -1535,6 +1542,12 @@ namespace boost a & x.m_multisig_info; a & x.m_multisig_k; a & x.m_key_image_partial; + if (ver < 10) + { + initialize_transfer_details(a, x, ver); + return; + } + a & x.m_key_image_requested; } template <class Archive> diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 5e6100dfd..50e3581f2 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -1781,7 +1781,7 @@ namespace tools try { - m_wallet->rescan_blockchain(); + m_wallet->rescan_blockchain(req.hard); } catch (const std::exception& e) { @@ -2460,12 +2460,13 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::vector<std::pair<crypto::key_image, crypto::signature>> ski = m_wallet->export_key_images(); - res.signed_key_images.resize(ski.size()); - for (size_t n = 0; n < ski.size(); ++n) + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(); + res.offset = ski.first; + res.signed_key_images.resize(ski.second.size()); + for (size_t n = 0; n < ski.second.size(); ++n) { - res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski[n].first); - res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski[n].second); + res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski.second[n].first); + res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski.second[n].second); } } @@ -2518,7 +2519,7 @@ namespace tools ski[n].second = *reinterpret_cast<const crypto::signature*>(bd.data()); } uint64_t spent = 0, unspent = 0; - uint64_t height = m_wallet->import_key_images(ski, spent, unspent); + uint64_t height = m_wallet->import_key_images(ski, req.offset, spent, unspent); res.spent = spent; res.unspent = unspent; res.height = height; @@ -3090,6 +3091,200 @@ namespace tools } } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request &req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response &res, epee::json_rpc::error &er) + { + if (m_wallet_dir.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_NO_WALLET_DIR; + er.message = "No wallet dir configured"; + return false; + } + + // early check for mandatory fields + if (req.filename.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; + return false; + } + if (req.seed.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'seed' is mandatory. Please provide a seed you want to restore from."; + return false; + } + + namespace po = boost::program_options; + po::variables_map vm2; + const char *ptr = strchr(req.filename.c_str(), '/'); + #ifdef _WIN32 + if (!ptr) + ptr = strchr(req.filename.c_str(), '\\'); + if (!ptr) + ptr = strchr(req.filename.c_str(), ':'); + #endif + if (ptr) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Invalid filename"; + return false; + } + std::string wallet_file = m_wallet_dir + "/" + req.filename; + // check if wallet file already exists + if (!wallet_file.empty()) + { + try + { + boost::system::error_code ignored_ec; + THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(wallet_file, ignored_ec), error::file_exists, wallet_file); + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet already exists."; + return false; + } + } + crypto::secret_key recovery_key; + std::string old_language; + + // check the given seed + { + if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, old_language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Electrum-style word list failed verification"; + return false; + } + } + + // process seed_offset if given + { + if (!req.seed_offset.empty()) + { + recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset); + } + } + { + po::options_description desc("dummy"); + const command_line::arg_descriptor<std::string, true> arg_password = {"password", "password"}; + const char *argv[4]; + int argc = 3; + argv[0] = "wallet-rpc"; + argv[1] = "--password"; + argv[2] = req.password.c_str(); + argv[3] = NULL; + vm2 = *m_vm; + command_line::add_arg(desc, arg_password); + po::store(po::parse_command_line(argc, argv, desc), vm2); + } + + auto rc = tools::wallet2::make_new(vm2, true, nullptr); + std::unique_ptr<wallet2> wal; + wal = std::move(rc.first); + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to create wallet"; + return false; + } + + epee::wipeable_string password = rc.second.password(); + + bool was_deprecated_wallet = ((old_language == crypto::ElectrumWords::old_language_name) || + crypto::ElectrumWords::get_is_old_style_seed(req.seed)); + + std::string mnemonic_language = old_language; + if (was_deprecated_wallet) + { + // The user had used an older version of the wallet with old style mnemonics. + res.was_deprecated = true; + } + + if (old_language == crypto::ElectrumWords::old_language_name) + { + if (req.language.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet was using the old seed language. You need to specify a new seed language."; + return false; + } + std::vector<std::string> language_list; + std::vector<std::string> language_list_en; + crypto::ElectrumWords::get_language_list(language_list); + crypto::ElectrumWords::get_language_list(language_list_en, true); + if (std::find(language_list.begin(), language_list.end(), req.language) == language_list.end() && + std::find(language_list_en.begin(), language_list_en.end(), req.language) == language_list_en.end()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet was using the old seed language, and the specified new seed language is invalid."; + return false; + } + mnemonic_language = req.language; + } + + wal->set_seed_language(mnemonic_language); + + crypto::secret_key recovery_val; + try + { + recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false); + MINFO("Wallet has been restored.\n"); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + // // Convert the secret key back to seed + epee::wipeable_string electrum_words; + if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to encode seed"; + return false; + } + res.seed = electrum_words.data(); + + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to generate wallet"; + return false; + } + + // set blockheight if given + try + { + wal->set_refresh_from_block_height(req.restore_height); + wal->rewrite(wallet_file, password); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + } + m_wallet = wal.release(); + res.address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); + res.info = "Wallet has been restored successfully."; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 887723ed5..abbbe82c5 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -137,6 +137,7 @@ namespace tools MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) + MAP_JON_RPC_WE("restore_deterministic_wallet", on_restore_deterministic_wallet, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) @@ -216,6 +217,7 @@ namespace tools bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); + bool on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request& req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 924f3a0f1..afb8c6e91 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 5 +#define WALLET_RPC_VERSION_MINOR 6 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1057,7 +1057,10 @@ namespace wallet_rpc { struct request { + bool hard; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(hard, false); END_KV_SERIALIZE_MAP() }; @@ -1579,9 +1582,11 @@ namespace wallet_rpc struct response { + uint32_t offset; std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(offset); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1602,9 +1607,11 @@ namespace wallet_rpc struct request { + uint32_t offset; std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(offset, (uint32_t)0); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1924,6 +1931,43 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET + { + struct request + { + uint64_t restore_height; + std::string filename; + std::string seed; + std::string seed_offset; + std::string password; + std::string language; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(restore_height, (uint64_t)0) + KV_SERIALIZE(filename) + KV_SERIALIZE(seed) + KV_SERIALIZE(seed_offset) + KV_SERIALIZE(password) + KV_SERIALIZE(language) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string address; + std::string seed; + std::string info; + bool was_deprecated; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(seed) + KV_SERIALIZE(info) + KV_SERIALIZE(was_deprecated) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request diff --git a/tests/data/fuzz/cold-outputs/OUTPUTS2 b/tests/data/fuzz/cold-outputs/OUTPUTS2 Binary files differindex 907bcdb91..33cf39024 100644 --- a/tests/data/fuzz/cold-outputs/OUTPUTS2 +++ b/tests/data/fuzz/cold-outputs/OUTPUTS2 diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index 488a3b931..29b3ed267 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -77,7 +77,7 @@ int ColdOutputsFuzzer::run(const std::string &filename) s = std::string("\x01\x16serialization::archive") + s; try { - std::vector<tools::wallet2::transfer_details> outputs; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; std::stringstream iss; iss << s; boost::archive::portable_binary_iarchive ar(iss); diff --git a/tests/libwallet_api_tests/CMakeLists.txt b/tests/libwallet_api_tests/CMakeLists.txt index ef1b666ed..1a9cbc5a6 100644 --- a/tests/libwallet_api_tests/CMakeLists.txt +++ b/tests/libwallet_api_tests/CMakeLists.txt @@ -50,6 +50,8 @@ target_link_libraries(libwallet_api_tests ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} + ${Boost_LOCALE_LIBRARY} + ${ICU_LIBRARIES} ${GTEST_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) diff --git a/tests/performance_tests/ge_frombytes_vartime.h b/tests/performance_tests/ge_frombytes_vartime.h index ef9625d6b..3f7d55182 100644 --- a/tests/performance_tests/ge_frombytes_vartime.h +++ b/tests/performance_tests/ge_frombytes_vartime.h @@ -49,22 +49,29 @@ public: if (!base_class::init()) return false; + cryptonote::account_base m_alice; + cryptonote::transaction m_tx; + m_alice.generate(); std::vector<tx_destination_entry> destinations; destinations.push_back(tx_destination_entry(1, m_alice.get_keys().m_account_address, false)); - return construct_tx(this->m_miners[this->real_source_idx].get_keys(), this->m_sources, destinations, boost::none, std::vector<uint8_t>(), m_tx, 0); + if (!construct_tx(this->m_miners[this->real_source_idx].get_keys(), this->m_sources, destinations, boost::none, std::vector<uint8_t>(), m_tx, 0)) + return false; + + const cryptonote::txin_to_key& txin = boost::get<cryptonote::txin_to_key>(m_tx.vin[0]); + m_key = rct::ki2rct(txin.k_image); + + return true; } bool test() { ge_p3 unp; - const cryptonote::txin_to_key& txin = boost::get<cryptonote::txin_to_key>(m_tx.vin[0]); - return ge_frombytes_vartime(&unp, (const unsigned char*) &txin.k_image) == 0; + return ge_frombytes_vartime(&unp, (const unsigned char*) &m_key) == 0; } private: - cryptonote::account_base m_alice; - cryptonote::transaction m_tx; + rct::key m_key; }; diff --git a/tests/performance_tests/ge_tobytes.h b/tests/performance_tests/ge_tobytes.h new file mode 100644 index 000000000..3d46f4838 --- /dev/null +++ b/tests/performance_tests/ge_tobytes.h @@ -0,0 +1,79 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_basic.h" + +#include "single_tx_test_base.h" + +class test_ge_tobytes : public multi_tx_test_base<1> +{ +public: + static const size_t loop_count = 10000; + + typedef multi_tx_test_base<1> base_class; + + bool init() + { + using namespace cryptonote; + + if (!base_class::init()) + return false; + + cryptonote::account_base m_alice; + cryptonote::transaction m_tx; + + m_alice.generate(); + + std::vector<tx_destination_entry> destinations; + destinations.push_back(tx_destination_entry(1, m_alice.get_keys().m_account_address, false)); + + if (!construct_tx(this->m_miners[this->real_source_idx].get_keys(), this->m_sources, destinations, boost::none, std::vector<uint8_t>(), m_tx, 0)) + return false; + + const cryptonote::txin_to_key& txin = boost::get<cryptonote::txin_to_key>(m_tx.vin[0]); + if (ge_frombytes_vartime(&m_p3, (const unsigned char*) &txin.k_image) != 0) + return false; + + return true; + } + + bool test() + { + rct::key key; + ge_p3_tobytes(key.bytes, &m_p3); + return true; + } + +private: + ge_p3 m_p3; +}; diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 3765d1249..58daf6220 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -42,6 +42,7 @@ #include "derive_public_key.h" #include "derive_secret_key.h" #include "ge_frombytes_vartime.h" +#include "ge_tobytes.h" #include "generate_key_derivation.h" #include "generate_key_image.h" #include "generate_key_image_helper.h" @@ -183,6 +184,7 @@ int main(int argc, char** argv) TEST_PERFORMANCE0(filter, p, test_derive_public_key); TEST_PERFORMANCE0(filter, p, test_derive_secret_key); TEST_PERFORMANCE0(filter, p, test_ge_frombytes_vartime); + TEST_PERFORMANCE0(filter, p, test_ge_tobytes); TEST_PERFORMANCE0(filter, p, test_generate_keypair); TEST_PERFORMANCE0(filter, p, test_sc_reduce32); TEST_PERFORMANCE0(filter, p, test_sc_check); diff --git a/tests/unit_tests/keccak.cpp b/tests/unit_tests/keccak.cpp index 4276b0e1d..37da65d76 100644 --- a/tests/unit_tests/keccak.cpp +++ b/tests/unit_tests/keccak.cpp @@ -37,7 +37,7 @@ extern "C" { #define TEST_KECCAK(sz, chunks) \ std::string data; \ data.resize(sz); \ - for (size_t i = 0; i < sz; ++i) \ + for (size_t i = 0; i < data.size(); ++i) \ data[i] = i * 17; \ uint8_t md0[32], md1[32]; \ keccak((const uint8_t*)data.data(), data.size(), md0, 32); \ diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 2f7b5aac7..b4517af2f 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -908,6 +908,17 @@ TEST(Serialization, portability_outputs) ASSERT_TRUE(td2.m_pk_index == 0); } +struct unsigned_tx_set +{ + std::vector<tools::wallet2::tx_construction_data> txes; + tools::wallet2::transfer_container transfers; +}; +template <class Archive> +inline void serialize(Archive &a, unsigned_tx_set &x, const boost::serialization::version_type ver) +{ + a & x.txes; + a & x.transfers; +} #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\003" TEST(Serialization, portability_unsigned_tx) { @@ -918,7 +929,7 @@ TEST(Serialization, portability_unsigned_tx) ASSERT_TRUE(r); const size_t magiclen = strlen(UNSIGNED_TX_PREFIX); ASSERT_FALSE(strncmp(s.c_str(), UNSIGNED_TX_PREFIX, magiclen)); - tools::wallet2::unsigned_tx_set exported_txs; + unsigned_tx_set exported_txs; s = s.substr(magiclen); r = false; try |